diff --git "a/checkpoint-2259/trainer_state.json" "b/checkpoint-2259/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-2259/trainer_state.json" @@ -0,0 +1,17818 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9563928873835732, + "global_step": 2259, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 3.0643, + "theoretical_loss": 3.321567680436603, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 3.0798, + "theoretical_loss": 3.3215564803546, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 2.8688, + "theoretical_loss": 3.321545280900887, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 2.7194, + "theoretical_loss": 3.3215340820754022, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 2.6193, + "theoretical_loss": 3.3215228838780817, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025, + "loss": 2.8571, + "theoretical_loss": 3.3215116863088636, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002916666666666667, + "loss": 2.7571, + "theoretical_loss": 3.3215004893676854, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003333333333333333, + "loss": 2.8877, + "theoretical_loss": 3.321489293054483, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.0, + "learning_rate": 0.000375, + "loss": 2.714, + "theoretical_loss": 3.321478097369195, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.0, + "learning_rate": 0.0004166666666666667, + "loss": 2.6564, + "theoretical_loss": 3.321466902311758, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0004583333333333333, + "loss": 2.5638, + "theoretical_loss": 3.3214557078821096, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 2.4354, + "theoretical_loss": 3.321444514080187, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 1640856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5749809741973877, + "objective/train/theoretical_loss": 3.321438917414603, + "objective/train/tokens_used": 22097376, + "theoretical_loss": 3.321438917414603, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005416666666666666, + "loss": 2.5713, + "theoretical_loss": 3.321433320905927, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005833333333333334, + "loss": 2.4812, + "theoretical_loss": 3.3214221283592678, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.01, + "learning_rate": 0.000625, + "loss": 2.7622, + "theoretical_loss": 3.321410936440146, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006666666666666666, + "loss": 2.609, + "theoretical_loss": 3.3213997451485, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007083333333333334, + "loss": 2.4537, + "theoretical_loss": 3.3213885544842654, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00075, + "loss": 2.4831, + "theoretical_loss": 3.321377364447381, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007916666666666666, + "loss": 2.5607, + "theoretical_loss": 3.3213661750377836, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0008333333333333334, + "loss": 2.5759, + "theoretical_loss": 3.3213549862554106, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000875, + "loss": 2.3106, + "theoretical_loss": 3.3213437981001994, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009166666666666666, + "loss": 2.5471, + "theoretical_loss": 3.3213326105720875, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009583333333333334, + "loss": 2.602, + "theoretical_loss": 3.3213214236710122, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.01, + "learning_rate": 0.001, + "loss": 2.5877, + "theoretical_loss": 3.321310237396911, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 1641461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.557373523712158, + "objective/train/theoretical_loss": 3.3212990517497207, + "objective/train/tokens_used": 23735776, + "theoretical_loss": 3.3212990517497207, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009995722840034217, + "loss": 2.7042, + "theoretical_loss": 3.3212990517497207, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009991445680068436, + "loss": 2.5234, + "theoretical_loss": 3.3212878667293797, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009987168520102653, + "loss": 2.5502, + "theoretical_loss": 3.321276682335825, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000998289136013687, + "loss": 2.6849, + "theoretical_loss": 3.3212654985689936, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009978614200171086, + "loss": 2.6348, + "theoretical_loss": 3.3212543154288237, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009974337040205303, + "loss": 2.6793, + "theoretical_loss": 3.3212431329152525, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009970059880239522, + "loss": 2.6212, + "theoretical_loss": 3.321231951028217, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009965782720273739, + "loss": 2.629, + "theoretical_loss": 3.3212207697676552, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009961505560307955, + "loss": 2.5865, + "theoretical_loss": 3.3212095891335043, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0009957228400342174, + "loss": 2.667, + "theoretical_loss": 3.321198409125702, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000995295124037639, + "loss": 2.55, + "theoretical_loss": 3.321187229744186, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009948674080410608, + "loss": 2.6324, + "theoretical_loss": 3.321176050988893, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009944396920444824, + "loss": 2.8406, + "theoretical_loss": 3.3211648728597614, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 1642666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8021185398101807, + "objective/train/theoretical_loss": 3.3211592840299864, + "objective/train/tokens_used": 25374176, + "theoretical_loss": 3.3211592840299864, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009940119760479041, + "loss": 2.5438, + "theoretical_loss": 3.3211536953567284, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000993584260051326, + "loss": 2.7618, + "theoretical_loss": 3.321142518479731, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009931565440547477, + "loss": 2.6638, + "theoretical_loss": 3.321131342228708, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009927288280581694, + "loss": 2.7287, + "theoretical_loss": 3.321120166603596, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000992301112061591, + "loss": 2.5832, + "theoretical_loss": 3.3211089916043326, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009918733960650127, + "loss": 2.6355, + "theoretical_loss": 3.3210978172308554, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009914456800684346, + "loss": 2.5139, + "theoretical_loss": 3.3210866434831026, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009910179640718563, + "loss": 2.5879, + "theoretical_loss": 3.3210754703610106, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009905902480752782, + "loss": 2.5935, + "theoretical_loss": 3.321064297864518, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009901625320786998, + "loss": 2.6176, + "theoretical_loss": 3.3210531259935627, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009897348160821215, + "loss": 2.6405, + "theoretical_loss": 3.321041954748081, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009893071000855432, + "loss": 2.7274, + "theoretical_loss": 3.321030784128012, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 1643300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.023165464401245, + "objective/train/theoretical_loss": 3.321019614133292, + "objective/train/tokens_used": 27012576, + "theoretical_loss": 3.321019614133292, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009888793840889649, + "loss": 2.7003, + "theoretical_loss": 3.321019614133292, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009884516680923865, + "loss": 2.7436, + "theoretical_loss": 3.3210084447638595, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009880239520958084, + "loss": 2.5873, + "theoretical_loss": 3.320997276019652, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.02, + "learning_rate": 0.00098759623609923, + "loss": 2.5974, + "theoretical_loss": 3.3209861079006067, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000987168520102652, + "loss": 2.5806, + "theoretical_loss": 3.320974940406662, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009867408041060737, + "loss": 2.6771, + "theoretical_loss": 3.320963773537755, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009863130881094953, + "loss": 2.7313, + "theoretical_loss": 3.320952607293824, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.02, + "learning_rate": 0.000985885372112917, + "loss": 2.7302, + "theoretical_loss": 3.320941441674806, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009854576561163387, + "loss": 2.893, + "theoretical_loss": 3.320930276680639, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0009850299401197606, + "loss": 2.6886, + "theoretical_loss": 3.3209191123112607, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009846022241231823, + "loss": 2.7801, + "theoretical_loss": 3.320907948566609, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000984174508126604, + "loss": 2.6538, + "theoretical_loss": 3.3208967854466214, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009837467921300258, + "loss": 2.5705, + "theoretical_loss": 3.3208856229512356, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 1644380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.542499542236328, + "objective/train/theoretical_loss": 3.320880041937749, + "objective/train/tokens_used": 28650976, + "theoretical_loss": 3.320880041937749, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009833190761334475, + "loss": 2.5074, + "theoretical_loss": 3.3208744610803898, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009828913601368692, + "loss": 2.6151, + "theoretical_loss": 3.320863299834021, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009824636441402908, + "loss": 2.6994, + "theoretical_loss": 3.320852139212068, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009820359281437125, + "loss": 2.568, + "theoretical_loss": 3.3208409792144677, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009816082121471344, + "loss": 2.5552, + "theoretical_loss": 3.320829819841158, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.03, + "learning_rate": 0.000981180496150556, + "loss": 2.6719, + "theoretical_loss": 3.320818661092077, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009807527801539778, + "loss": 2.5567, + "theoretical_loss": 3.3208075029671624, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009803250641573994, + "loss": 2.5511, + "theoretical_loss": 3.320796345466352, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009798973481608211, + "loss": 2.6608, + "theoretical_loss": 3.320785188589584, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000979469632164243, + "loss": 2.5947, + "theoretical_loss": 3.3207740323367956, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009790419161676647, + "loss": 2.6511, + "theoretical_loss": 3.3207628767079242, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009786142001710863, + "loss": 2.5573, + "theoretical_loss": 3.3207517217029094, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 1645056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4763801097869873, + "objective/train/theoretical_loss": 3.3207405673216877, + "objective/train/tokens_used": 30289376, + "theoretical_loss": 3.3207405673216877, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009781864841745082, + "loss": 2.6745, + "theoretical_loss": 3.3207405673216877, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.03, + "learning_rate": 0.00097775876817793, + "loss": 2.697, + "theoretical_loss": 3.320729413564197, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009773310521813516, + "loss": 2.6853, + "theoretical_loss": 3.3207182604303753, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009769033361847733, + "loss": 2.3445, + "theoretical_loss": 3.320707107920161, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.03, + "learning_rate": 0.000976475620188195, + "loss": 2.6763, + "theoretical_loss": 3.3206959560334917, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009760479041916168, + "loss": 2.5198, + "theoretical_loss": 3.320684804770305, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009756201881950385, + "loss": 2.7, + "theoretical_loss": 3.3206736541305393, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0009751924721984602, + "loss": 2.6958, + "theoretical_loss": 3.3206625041141318, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.04, + "learning_rate": 0.000974764756201882, + "loss": 2.6457, + "theoretical_loss": 3.3206513547210212, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009743370402053036, + "loss": 2.7946, + "theoretical_loss": 3.320640205951145, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009739093242087254, + "loss": 2.6682, + "theoretical_loss": 3.3206290578044415, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009734816082121472, + "loss": 2.5484, + "theoretical_loss": 3.3206179102808484, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009730538922155689, + "loss": 2.6724, + "theoretical_loss": 3.3206067633803036, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 1646327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547891855239868, + "objective/train/theoretical_loss": 3.320601190163655, + "objective/train/tokens_used": 31927776, + "theoretical_loss": 3.320601190163655, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009726261762189907, + "loss": 2.5267, + "theoretical_loss": 3.320595617102745, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009721984602224123, + "loss": 2.6548, + "theoretical_loss": 3.320584471448111, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009717707442258341, + "loss": 2.5147, + "theoretical_loss": 3.3205733264163393, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009713430282292558, + "loss": 2.4505, + "theoretical_loss": 3.320562182007368, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009709153122326775, + "loss": 2.6305, + "theoretical_loss": 3.320551038221135, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009704875962360993, + "loss": 2.5482, + "theoretical_loss": 3.3205398950575784, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009700598802395209, + "loss": 2.7266, + "theoretical_loss": 3.320528752516636, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009696321642429427, + "loss": 2.5155, + "theoretical_loss": 3.3205176105982463, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009692044482463645, + "loss": 2.7628, + "theoretical_loss": 3.320506469302347, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009687767322497862, + "loss": 2.6802, + "theoretical_loss": 3.3204953286288763, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.04, + "learning_rate": 0.000968349016253208, + "loss": 2.7921, + "theoretical_loss": 3.3204841885777725, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009679213002566296, + "loss": 2.6088, + "theoretical_loss": 3.3204730491489727, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 1647543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5402848720550537, + "objective/train/theoretical_loss": 3.3204619103424164, + "objective/train/tokens_used": 33566176, + "theoretical_loss": 3.3204619103424164, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009674935842600513, + "loss": 2.6446, + "theoretical_loss": 3.3204619103424164, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009670658682634731, + "loss": 2.5202, + "theoretical_loss": 3.3204507721580403, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009666381522668948, + "loss": 2.4832, + "theoretical_loss": 3.3204396345957834, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009662104362703165, + "loss": 2.6544, + "theoretical_loss": 3.320428497655584, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009657827202737382, + "loss": 2.7507, + "theoretical_loss": 3.320417361337379, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00096535500427716, + "loss": 2.5786, + "theoretical_loss": 3.3204062256411078, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0009649272882805818, + "loss": 2.6649, + "theoretical_loss": 3.320395090566708, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009644995722840035, + "loss": 2.6338, + "theoretical_loss": 3.3203839561141173, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009640718562874252, + "loss": 2.664, + "theoretical_loss": 3.320372822283275, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009636441402908469, + "loss": 2.6909, + "theoretical_loss": 3.3203616890741183, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009632164242942686, + "loss": 2.4478, + "theoretical_loss": 3.3203505564865856, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009627887082976904, + "loss": 2.6265, + "theoretical_loss": 3.3203394245206153, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.05, + "learning_rate": 0.000962360992301112, + "loss": 2.5755, + "theoretical_loss": 3.320328293176145, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 1648109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5592408180236816, + "objective/train/theoretical_loss": 3.3203227277369534, + "objective/train/tokens_used": 35204576, + "theoretical_loss": 3.3203227277369534, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009619332763045337, + "loss": 2.567, + "theoretical_loss": 3.320317162453114, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009615055603079555, + "loss": 2.6035, + "theoretical_loss": 3.3203060323514593, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009610778443113773, + "loss": 2.6209, + "theoretical_loss": 3.3202949028711197, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009606501283147991, + "loss": 2.522, + "theoretical_loss": 3.3202837740120335, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009602224123182207, + "loss": 2.5764, + "theoretical_loss": 3.3202726457741387, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009597946963216424, + "loss": 2.6805, + "theoretical_loss": 3.320261518157374, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009593669803250642, + "loss": 2.4835, + "theoretical_loss": 3.3202503911616765, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009589392643284859, + "loss": 2.5907, + "theoretical_loss": 3.320239264786986, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009585115483319077, + "loss": 2.617, + "theoretical_loss": 3.3202281390332393, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009580838323353293, + "loss": 2.5027, + "theoretical_loss": 3.320217013900376, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000957656116338751, + "loss": 2.6857, + "theoretical_loss": 3.3202058893883333, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009572284003421729, + "loss": 2.6411, + "theoretical_loss": 3.3201947654970505, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 1649212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7101857662200928, + "objective/train/theoretical_loss": 3.320183642226465, + "objective/train/tokens_used": 36842976, + "theoretical_loss": 3.320183642226465, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009568006843455946, + "loss": 2.5369, + "theoretical_loss": 3.320183642226465, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009563729683490164, + "loss": 2.5836, + "theoretical_loss": 3.3201725195765155, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.05, + "learning_rate": 0.000955945252352438, + "loss": 2.5358, + "theoretical_loss": 3.3201613975471402, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009555175363558597, + "loss": 2.6035, + "theoretical_loss": 3.3201502761382775, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0009550898203592815, + "loss": 2.4594, + "theoretical_loss": 3.320139155349866, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009546621043627032, + "loss": 2.6376, + "theoretical_loss": 3.3201280351818436, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009542343883661248, + "loss": 2.5904, + "theoretical_loss": 3.320116915634149, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009538066723695466, + "loss": 2.7616, + "theoretical_loss": 3.3201057967067205, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009533789563729683, + "loss": 2.6076, + "theoretical_loss": 3.3200946783994962, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009529512403763902, + "loss": 2.6121, + "theoretical_loss": 3.3200835607124146, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009525235243798119, + "loss": 2.6083, + "theoretical_loss": 3.3200724436454143, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009520958083832335, + "loss": 2.6831, + "theoretical_loss": 3.3200613271984336, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009516680923866553, + "loss": 2.6156, + "theoretical_loss": 3.3200502113714108, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 1649940, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545426368713379, + "objective/train/theoretical_loss": 3.3200446536903643, + "objective/train/tokens_used": 38481376, + "theoretical_loss": 3.3200446536903643, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.06, + "learning_rate": 0.000951240376390077, + "loss": 2.6849, + "theoretical_loss": 3.3200390961642845, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009508126603934988, + "loss": 2.654, + "theoretical_loss": 3.3200279815769926, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009503849443969204, + "loss": 2.6215, + "theoretical_loss": 3.3200168676094743, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009499572284003421, + "loss": 2.4387, + "theoretical_loss": 3.320005754261668, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009495295124037639, + "loss": 2.5621, + "theoretical_loss": 3.319994641533511, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009491017964071857, + "loss": 2.731, + "theoretical_loss": 3.319983529424943, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009486740804106075, + "loss": 2.5847, + "theoretical_loss": 3.3199724179359027, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009482463644140291, + "loss": 2.5836, + "theoretical_loss": 3.319961307066327, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009478186484174508, + "loss": 2.6488, + "theoretical_loss": 3.3199501968161558, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009473909324208726, + "loss": 2.6244, + "theoretical_loss": 3.319939087185327, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009469632164242943, + "loss": 2.6873, + "theoretical_loss": 3.3199279781737796, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000946535500427716, + "loss": 2.5921, + "theoretical_loss": 3.3199168697814514, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 1651249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.582075834274292, + "objective/train/theoretical_loss": 3.3199057620082812, + "objective/train/tokens_used": 40119776, + "theoretical_loss": 3.3199057620082812, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009461077844311377, + "loss": 2.5527, + "theoretical_loss": 3.3199057620082812, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009456800684345594, + "loss": 2.7818, + "theoretical_loss": 3.319894654854208, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0009452523524379812, + "loss": 2.628, + "theoretical_loss": 3.3198835483191695, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000944824636441403, + "loss": 2.4649, + "theoretical_loss": 3.319872442403105, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009443969204448247, + "loss": 2.7571, + "theoretical_loss": 3.3198613371059524, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009439692044482464, + "loss": 2.5878, + "theoretical_loss": 3.319850232427651, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009435414884516681, + "loss": 2.6015, + "theoretical_loss": 3.3198391283681383, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009431137724550899, + "loss": 2.5823, + "theoretical_loss": 3.3198280249273546, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009426860564585116, + "loss": 2.7222, + "theoretical_loss": 3.319816922105237, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009422583404619332, + "loss": 2.6364, + "theoretical_loss": 3.319805819901724, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.07, + "learning_rate": 0.000941830624465355, + "loss": 2.6811, + "theoretical_loss": 3.3197947183167553, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009414029084687767, + "loss": 2.8016, + "theoretical_loss": 3.319783617350269, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009409751924721985, + "loss": 2.5657, + "theoretical_loss": 3.319772517002204, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1651905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0052103996276855, + "objective/train/theoretical_loss": 3.31976696706006, + "objective/train/tokens_used": 41758176, + "theoretical_loss": 3.31976696706006, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009405474764756203, + "loss": 2.6357, + "theoretical_loss": 3.319761417272498, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009401197604790419, + "loss": 2.5734, + "theoretical_loss": 3.319750318161091, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009396920444824637, + "loss": 2.424, + "theoretical_loss": 3.3197392196679205, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009392643284858854, + "loss": 2.5344, + "theoretical_loss": 3.3197281217929255, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009388366124893071, + "loss": 2.5689, + "theoretical_loss": 3.319717024536045, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009384088964927289, + "loss": 2.4989, + "theoretical_loss": 3.3197059278972176, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009379811804961505, + "loss": 2.6272, + "theoretical_loss": 3.3196948318763817, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009375534644995723, + "loss": 2.5959, + "theoretical_loss": 3.319683736473476, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000937125748502994, + "loss": 2.5228, + "theoretical_loss": 3.3196726416884395, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009366980325064158, + "loss": 2.7357, + "theoretical_loss": 3.3196615475212106, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009362703165098376, + "loss": 2.6946, + "theoretical_loss": 3.3196504539717284, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009358426005132592, + "loss": 2.7344, + "theoretical_loss": 3.3196393610399317, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1652881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5316507816314697, + "objective/train/theoretical_loss": 3.3196282687257583, + "objective/train/tokens_used": 43396576, + "theoretical_loss": 3.3196282687257583, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.07, + "learning_rate": 0.000935414884516681, + "loss": 2.7259, + "theoretical_loss": 3.3196282687257583, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009349871685201027, + "loss": 2.4882, + "theoretical_loss": 3.3196171770291483, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0009345594525235244, + "loss": 2.6169, + "theoretical_loss": 3.3196060859500394, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009341317365269461, + "loss": 2.5293, + "theoretical_loss": 3.319594995488371, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009337040205303678, + "loss": 2.7782, + "theoretical_loss": 3.3195839056440812, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009332763045337895, + "loss": 2.6719, + "theoretical_loss": 3.3195728164171094, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009328485885372114, + "loss": 2.7268, + "theoretical_loss": 3.319561727807394, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009324208725406331, + "loss": 2.5997, + "theoretical_loss": 3.3195506398148744, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009319931565440548, + "loss": 2.7602, + "theoretical_loss": 3.319539552439489, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009315654405474765, + "loss": 2.5845, + "theoretical_loss": 3.3195284656811763, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009311377245508982, + "loss": 2.7053, + "theoretical_loss": 3.319517379539876, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00093071000855432, + "loss": 2.6045, + "theoretical_loss": 3.3195062940155258, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009302822925577416, + "loss": 2.6324, + "theoretical_loss": 3.3194952091080654, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1653310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4821465015411377, + "objective/train/theoretical_loss": 3.3194896668856497, + "objective/train/tokens_used": 45034976, + "theoretical_loss": 3.3194896668856497, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009298545765611634, + "loss": 2.532, + "theoretical_loss": 3.3194841248174334, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009294268605645851, + "loss": 2.544, + "theoretical_loss": 3.3194730411435684, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009289991445680068, + "loss": 2.826, + "theoretical_loss": 3.3194619580864098, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009285714285714287, + "loss": 2.7561, + "theoretical_loss": 3.3194508756458965, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009281437125748503, + "loss": 2.5692, + "theoretical_loss": 3.319439793821967, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000927715996578272, + "loss": 2.6322, + "theoretical_loss": 3.3194287126145596, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009272882805816938, + "loss": 2.6346, + "theoretical_loss": 3.3194176320236144, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009268605645851155, + "loss": 2.7908, + "theoretical_loss": 3.31940655204907, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009264328485885373, + "loss": 2.6439, + "theoretical_loss": 3.319395472690865, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009260051325919589, + "loss": 2.5589, + "theoretical_loss": 3.3193843939489382, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009255774165953806, + "loss": 2.6926, + "theoretical_loss": 3.319373315823229, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009251497005988024, + "loss": 2.7646, + "theoretical_loss": 3.3193622383136763, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1654644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3390886783599854, + "objective/train/theoretical_loss": 3.3193511614202187, + "objective/train/tokens_used": 46673376, + "theoretical_loss": 3.3193511614202187, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0009247219846022242, + "loss": 2.5287, + "theoretical_loss": 3.3193511614202187, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.09, + "learning_rate": 0.000924294268605646, + "loss": 2.7607, + "theoretical_loss": 3.319340085142796, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009238665526090676, + "loss": 2.6289, + "theoretical_loss": 3.319329009481346, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009234388366124893, + "loss": 2.7648, + "theoretical_loss": 3.3193179344358086, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009230111206159111, + "loss": 2.497, + "theoretical_loss": 3.319306860006122, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009225834046193328, + "loss": 2.4963, + "theoretical_loss": 3.319295786192226, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009221556886227545, + "loss": 2.5823, + "theoretical_loss": 3.319284712994059, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009217279726261762, + "loss": 2.6555, + "theoretical_loss": 3.3192736404115606, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009213002566295979, + "loss": 2.5264, + "theoretical_loss": 3.3192625684446693, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009208725406330197, + "loss": 2.397, + "theoretical_loss": 3.3192514970933242, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009204448246364415, + "loss": 2.6273, + "theoretical_loss": 3.319240426357465, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009200171086398631, + "loss": 2.6414, + "theoretical_loss": 3.31922935623703, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009195893926432849, + "loss": 2.5706, + "theoretical_loss": 3.3192182867319584, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1655335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3051180839538574, + "objective/train/theoretical_loss": 3.319212752210165, + "objective/train/tokens_used": 48311776, + "theoretical_loss": 3.319212752210165, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009191616766467066, + "loss": 2.5393, + "theoretical_loss": 3.3192072178421896, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009187339606501284, + "loss": 2.6545, + "theoretical_loss": 3.319196149567662, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00091830624465355, + "loss": 2.5623, + "theoretical_loss": 3.3191850819083157, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009178785286569717, + "loss": 2.6804, + "theoretical_loss": 3.319174014864089, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009174508126603935, + "loss": 2.8051, + "theoretical_loss": 3.319162948434921, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009170230966638152, + "loss": 2.7044, + "theoretical_loss": 3.319151882620752, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009165953806672371, + "loss": 2.5421, + "theoretical_loss": 3.3191408174215193, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009161676646706587, + "loss": 2.7474, + "theoretical_loss": 3.3191297528371635, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009157399486740804, + "loss": 2.5804, + "theoretical_loss": 3.319118688867623, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009153122326775022, + "loss": 2.5145, + "theoretical_loss": 3.319107625512837, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009148845166809239, + "loss": 2.746, + "theoretical_loss": 3.3190965627727445, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0009144568006843457, + "loss": 2.5949, + "theoretical_loss": 3.3190855006472857, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1656670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1859257221221924, + "objective/train/theoretical_loss": 3.3190744391363984, + "objective/train/tokens_used": 49950176, + "theoretical_loss": 3.3190744391363984, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009140290846877673, + "loss": 2.6334, + "theoretical_loss": 3.3190744391363984, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.1, + "learning_rate": 0.000913601368691189, + "loss": 2.6976, + "theoretical_loss": 3.3190633782400223, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009131736526946108, + "loss": 2.5658, + "theoretical_loss": 3.3190523179580973, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009127459366980325, + "loss": 2.7495, + "theoretical_loss": 3.3190412582905617, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009123182207014543, + "loss": 2.7093, + "theoretical_loss": 3.319030199237355, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000911890504704876, + "loss": 2.5295, + "theoretical_loss": 3.3190191407984164, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009114627887082977, + "loss": 2.6081, + "theoretical_loss": 3.3190080829736854, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009110350727117195, + "loss": 2.6166, + "theoretical_loss": 3.318997025763101, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009106073567151412, + "loss": 2.7691, + "theoretical_loss": 3.318985969166602, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009101796407185628, + "loss": 2.6465, + "theoretical_loss": 3.3189749131841286, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009097519247219846, + "loss": 2.5314, + "theoretical_loss": 3.3189638578156195, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009093242087254063, + "loss": 2.8726, + "theoretical_loss": 3.3189528030610136, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009088964927288281, + "loss": 2.6762, + "theoretical_loss": 3.318941748920251, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1657192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9018635749816895, + "objective/train/theoretical_loss": 3.318936222080042, + "objective/train/tokens_used": 51588576, + "theoretical_loss": 3.318936222080042, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009084687767322499, + "loss": 2.643, + "theoretical_loss": 3.318930695393271, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009080410607356715, + "loss": 2.6665, + "theoretical_loss": 3.3189196424800116, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009076133447390933, + "loss": 2.6265, + "theoretical_loss": 3.3189085901804134, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.1, + "learning_rate": 0.000907185628742515, + "loss": 2.5788, + "theoretical_loss": 3.3188975384944155, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009067579127459367, + "loss": 2.6531, + "theoretical_loss": 3.318886487421957, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009063301967493585, + "loss": 2.7177, + "theoretical_loss": 3.318875436962977, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009059024807527801, + "loss": 2.4989, + "theoretical_loss": 3.3188643871174155, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009054747647562019, + "loss": 2.5249, + "theoretical_loss": 3.318853337885211, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009050470487596236, + "loss": 2.6031, + "theoretical_loss": 3.318842289266304, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009046193327630453, + "loss": 2.5702, + "theoretical_loss": 3.3188312412606327, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0009041916167664672, + "loss": 2.5348, + "theoretical_loss": 3.3188201938681368, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009037639007698888, + "loss": 2.5876, + "theoretical_loss": 3.318809147088756, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1658380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8608322143554688, + "objective/train/theoretical_loss": 3.3187981009224297, + "objective/train/tokens_used": 53226976, + "theoretical_loss": 3.3187981009224297, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009033361847733106, + "loss": 2.6644, + "theoretical_loss": 3.3187981009224297, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009029084687767323, + "loss": 2.5824, + "theoretical_loss": 3.3187870553690972, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.11, + "learning_rate": 0.000902480752780154, + "loss": 2.7377, + "theoretical_loss": 3.3187760104286976, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009020530367835757, + "loss": 2.6572, + "theoretical_loss": 3.3187649661011704, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009016253207869974, + "loss": 2.5447, + "theoretical_loss": 3.3187539223864557, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009011976047904192, + "loss": 2.6011, + "theoretical_loss": 3.318742879284492, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009007698887938409, + "loss": 2.4863, + "theoretical_loss": 3.3187318367952194, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0009003421727972627, + "loss": 2.4994, + "theoretical_loss": 3.318720794918577, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008999144568006844, + "loss": 2.5443, + "theoretical_loss": 3.3187097536545047, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008994867408041061, + "loss": 2.6808, + "theoretical_loss": 3.3186987130029415, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008990590248075278, + "loss": 2.7511, + "theoretical_loss": 3.3186876729638266, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008986313088109496, + "loss": 2.612, + "theoretical_loss": 3.3186766335371005, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008982035928143712, + "loss": 2.686, + "theoretical_loss": 3.318665594722702, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1658975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8614556789398193, + "objective/train/theoretical_loss": 3.3186600755451066, + "objective/train/tokens_used": 54865376, + "theoretical_loss": 3.3186600755451066, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000897775876817793, + "loss": 2.654, + "theoretical_loss": 3.3186545565205705, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008973481608212147, + "loss": 2.6409, + "theoretical_loss": 3.318643518930646, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008969204448246364, + "loss": 2.5854, + "theoretical_loss": 3.3186324819528674, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008964927288280582, + "loss": 2.6063, + "theoretical_loss": 3.318621445587175, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008960650128314799, + "loss": 2.537, + "theoretical_loss": 3.3186104098335076, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008956372968349017, + "loss": 2.5978, + "theoretical_loss": 3.318599374691805, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008952095808383234, + "loss": 2.6201, + "theoretical_loss": 3.318588340162007, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008947818648417451, + "loss": 2.7451, + "theoretical_loss": 3.3185773062440527, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0008943541488451669, + "loss": 2.5093, + "theoretical_loss": 3.3185662729378826, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008939264328485885, + "loss": 2.517, + "theoretical_loss": 3.318555240243435, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008934987168520102, + "loss": 2.515, + "theoretical_loss": 3.3185442081606507, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.12, + "learning_rate": 0.000893071000855432, + "loss": 2.6143, + "theoretical_loss": 3.3185331766894683, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1660130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4469985961914062, + "objective/train/theoretical_loss": 3.318522145829828, + "objective/train/tokens_used": 56503776, + "theoretical_loss": 3.318522145829828, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008926432848588537, + "loss": 2.4787, + "theoretical_loss": 3.318522145829828, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008922155688622756, + "loss": 2.6368, + "theoretical_loss": 3.318511115581669, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008917878528656972, + "loss": 2.6092, + "theoretical_loss": 3.3185000859449314, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008913601368691189, + "loss": 2.5731, + "theoretical_loss": 3.3184890569195544, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008909324208725407, + "loss": 2.5214, + "theoretical_loss": 3.3184780285054782, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008905047048759624, + "loss": 2.5803, + "theoretical_loss": 3.318467000702642, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008900769888793841, + "loss": 2.6223, + "theoretical_loss": 3.3184559735109853, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008896492728828058, + "loss": 2.5938, + "theoretical_loss": 3.3184449469304482, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008892215568862275, + "loss": 2.6878, + "theoretical_loss": 3.3184339209609703, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008887938408896493, + "loss": 2.4932, + "theoretical_loss": 3.318422895602491, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.12, + "learning_rate": 0.000888366124893071, + "loss": 2.4597, + "theoretical_loss": 3.31841187085495, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008879384088964928, + "loss": 2.4485, + "theoretical_loss": 3.318400846718288, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008875106928999145, + "loss": 2.6036, + "theoretical_loss": 3.318389823192443, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1661412, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3641762733459473, + "objective/train/theoretical_loss": 3.3183843116585585, + "objective/train/tokens_used": 58142176, + "theoretical_loss": 3.3183843116585585, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008870829769033362, + "loss": 2.6277, + "theoretical_loss": 3.318378800277356, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.12, + "learning_rate": 0.000886655260906758, + "loss": 2.5149, + "theoretical_loss": 3.3183677779729663, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008862275449101797, + "loss": 2.6112, + "theoretical_loss": 3.3183567562792136, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008857998289136013, + "loss": 2.6564, + "theoretical_loss": 3.3183457351960377, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008853721129170231, + "loss": 2.6973, + "theoretical_loss": 3.3183347147233784, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008849443969204448, + "loss": 2.4547, + "theoretical_loss": 3.3183236948611756, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008845166809238666, + "loss": 2.3115, + "theoretical_loss": 3.3183126756093686, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0008840889649272883, + "loss": 2.5859, + "theoretical_loss": 3.318301656967898, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00088366124893071, + "loss": 2.4681, + "theoretical_loss": 3.3182906389367024, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008832335329341318, + "loss": 2.4459, + "theoretical_loss": 3.3182796215157224, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008828058169375535, + "loss": 2.5877, + "theoretical_loss": 3.318268604704898, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008823781009409753, + "loss": 2.5306, + "theoretical_loss": 3.318257588504168, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1662079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3965446949005127, + "objective/train/theoretical_loss": 3.318246572913474, + "objective/train/tokens_used": 59780576, + "theoretical_loss": 3.318246572913474, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008819503849443969, + "loss": 2.6399, + "theoretical_loss": 3.318246572913474, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008815226689478186, + "loss": 2.6156, + "theoretical_loss": 3.318235557932754, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008810949529512404, + "loss": 2.4887, + "theoretical_loss": 3.318224543561948, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008806672369546621, + "loss": 2.7212, + "theoretical_loss": 3.3182135298009974, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008802395209580839, + "loss": 2.6808, + "theoretical_loss": 3.3182025166498406, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008798118049615056, + "loss": 2.4704, + "theoretical_loss": 3.3181915041084182, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008793840889649273, + "loss": 2.4549, + "theoretical_loss": 3.3181804921766695, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008789563729683491, + "loss": 2.8214, + "theoretical_loss": 3.318169480854535, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008785286569717708, + "loss": 2.5934, + "theoretical_loss": 3.318158470141954, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008781009409751924, + "loss": 2.6187, + "theoretical_loss": 3.3181474600388667, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008776732249786142, + "loss": 2.3858, + "theoretical_loss": 3.318136450545213, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008772455089820359, + "loss": 2.5896, + "theoretical_loss": 3.318125441660933, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008768177929854577, + "loss": 2.388, + "theoretical_loss": 3.318114433385966, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1662642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5412111282348633, + "objective/train/theoretical_loss": 3.3181089294769563, + "objective/train/tokens_used": 61418976, + "theoretical_loss": 3.3181089294769563, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008763900769888794, + "loss": 2.6377, + "theoretical_loss": 3.3181034257202526, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008759623609923011, + "loss": 2.55, + "theoretical_loss": 3.318092418663732, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008755346449957229, + "loss": 2.6605, + "theoretical_loss": 3.3180814122163453, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008751069289991446, + "loss": 2.6276, + "theoretical_loss": 3.3180704063780313, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008746792130025664, + "loss": 2.6145, + "theoretical_loss": 3.318059401148731, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0008742514970059881, + "loss": 2.7501, + "theoretical_loss": 3.3180483965283836, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008738237810094097, + "loss": 2.4986, + "theoretical_loss": 3.318037392516929, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008733960650128315, + "loss": 2.5994, + "theoretical_loss": 3.318026389114308, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008729683490162532, + "loss": 2.4541, + "theoretical_loss": 3.3180153863204596, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008725406330196749, + "loss": 2.6821, + "theoretical_loss": 3.3180043841353246, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008721129170230966, + "loss": 2.5988, + "theoretical_loss": 3.317993382558843, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008716852010265184, + "loss": 2.6567, + "theoretical_loss": 3.317982381590954, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1663221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.975888252258301, + "objective/train/theoretical_loss": 3.3179713812315983, + "objective/train/tokens_used": 63057376, + "theoretical_loss": 3.3179713812315983, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008712574850299402, + "loss": 2.6197, + "theoretical_loss": 3.3179713812315983, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008708297690333619, + "loss": 2.7088, + "theoretical_loss": 3.317960381480716, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008704020530367836, + "loss": 2.582, + "theoretical_loss": 3.317949382338247, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008699743370402053, + "loss": 2.5966, + "theoretical_loss": 3.3179383838041314, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.14, + "learning_rate": 0.000869546621043627, + "loss": 2.8364, + "theoretical_loss": 3.317927385878309, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008691189050470488, + "loss": 2.6666, + "theoretical_loss": 3.31791638856072, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008686911890504705, + "loss": 2.6976, + "theoretical_loss": 3.317905391851305, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008682634730538922, + "loss": 2.662, + "theoretical_loss": 3.3178943957500033, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.14, + "learning_rate": 0.000867835757057314, + "loss": 2.7223, + "theoretical_loss": 3.317883400256756, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008674080410607357, + "loss": 2.754, + "theoretical_loss": 3.3178724053715016, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008669803250641575, + "loss": 2.6831, + "theoretical_loss": 3.3178614110941815, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008665526090675792, + "loss": 2.8167, + "theoretical_loss": 3.3178504174247356, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008661248930710008, + "loss": 2.761, + "theoretical_loss": 3.317839424363104, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1664363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568387746810913, + "objective/train/theoretical_loss": 3.3178339280602, + "objective/train/tokens_used": 64695776, + "theoretical_loss": 3.3178339280602, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008656971770744226, + "loss": 2.5995, + "theoretical_loss": 3.317828431909227, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008652694610778443, + "loss": 2.5506, + "theoretical_loss": 3.3178174400630445, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.14, + "learning_rate": 0.000864841745081266, + "loss": 2.8138, + "theoretical_loss": 3.3178064488244967, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008644140290846878, + "loss": 2.5913, + "theoretical_loss": 3.3177954581935234, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0008639863130881094, + "loss": 2.5688, + "theoretical_loss": 3.317784468170066, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008635585970915313, + "loss": 2.5422, + "theoretical_loss": 3.317773478754063, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.15, + "learning_rate": 0.000863130881094953, + "loss": 2.6193, + "theoretical_loss": 3.317762489945456, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008627031650983747, + "loss": 2.5738, + "theoretical_loss": 3.3177515017441843, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008622754491017965, + "loss": 2.715, + "theoretical_loss": 3.3177405141501883, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008618477331052181, + "loss": 2.643, + "theoretical_loss": 3.317729527163409, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008614200171086399, + "loss": 2.987, + "theoretical_loss": 3.3177185407837855, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008609923011120616, + "loss": 2.594, + "theoretical_loss": 3.3177075550112587, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1664891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.584660291671753, + "objective/train/theoretical_loss": 3.3176965698457686, + "objective/train/tokens_used": 66334176, + "theoretical_loss": 3.3176965698457686, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008605645851154833, + "loss": 2.6228, + "theoretical_loss": 3.3176965698457686, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.15, + "learning_rate": 0.000860136869118905, + "loss": 2.6502, + "theoretical_loss": 3.3176855852872555, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008597091531223268, + "loss": 2.6091, + "theoretical_loss": 3.3176746013356597, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008592814371257485, + "loss": 2.5742, + "theoretical_loss": 3.317663617990922, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008588537211291703, + "loss": 2.573, + "theoretical_loss": 3.3176526352529816, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.15, + "learning_rate": 0.000858426005132592, + "loss": 2.6126, + "theoretical_loss": 3.31764165312178, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008579982891360137, + "loss": 2.6855, + "theoretical_loss": 3.3176306715972563, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008575705731394354, + "loss": 2.7764, + "theoretical_loss": 3.3176196906793516, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008571428571428571, + "loss": 2.625, + "theoretical_loss": 3.3176087103680056, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008567151411462789, + "loss": 2.6693, + "theoretical_loss": 3.3175977306631594, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008562874251497006, + "loss": 2.6201, + "theoretical_loss": 3.3175867515647526, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008558597091531223, + "loss": 2.5103, + "theoretical_loss": 3.317575773072726, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008554319931565441, + "loss": 2.4253, + "theoretical_loss": 3.3175647951870197, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1665791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2302393913269043, + "objective/train/theoretical_loss": 3.317559306471518, + "objective/train/tokens_used": 67972576, + "theoretical_loss": 3.317559306471518, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008550042771599658, + "loss": 2.6287, + "theoretical_loss": 3.3175538179075743, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008545765611633876, + "loss": 2.5721, + "theoretical_loss": 3.31754284123433, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0008541488451668093, + "loss": 2.5295, + "theoretical_loss": 3.3175318651672274, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.15, + "learning_rate": 0.000853721129170231, + "loss": 2.6531, + "theoretical_loss": 3.3175208897062065, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008532934131736527, + "loss": 2.7652, + "theoretical_loss": 3.317509914851208, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008528656971770744, + "loss": 2.4813, + "theoretical_loss": 3.3174989406021718, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008524379811804962, + "loss": 2.7802, + "theoretical_loss": 3.317487966959039, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008520102651839178, + "loss": 2.6504, + "theoretical_loss": 3.3174769939217494, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008515825491873395, + "loss": 2.6791, + "theoretical_loss": 3.317466021490244, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008511548331907614, + "loss": 2.8812, + "theoretical_loss": 3.3174550496644626, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008507271171941831, + "loss": 2.6779, + "theoretical_loss": 3.317444078444346, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008502994011976049, + "loss": 2.6431, + "theoretical_loss": 3.317433107829835, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1666368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.657705307006836, + "objective/train/theoretical_loss": 3.31742213782087, + "objective/train/tokens_used": 69610976, + "theoretical_loss": 3.31742213782087, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008498716852010265, + "loss": 2.72, + "theoretical_loss": 3.31742213782087, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008494439692044482, + "loss": 2.589, + "theoretical_loss": 3.3174111684173906, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00084901625320787, + "loss": 2.7118, + "theoretical_loss": 3.317400199619338, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008485885372112917, + "loss": 2.6093, + "theoretical_loss": 3.3173892314266524, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008481608212147135, + "loss": 2.724, + "theoretical_loss": 3.3173782638392746, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008477331052181351, + "loss": 2.6337, + "theoretical_loss": 3.3173672968571446, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008473053892215569, + "loss": 2.7281, + "theoretical_loss": 3.3173563304802034, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008468776732249787, + "loss": 2.6731, + "theoretical_loss": 3.3173453647083915, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008464499572284004, + "loss": 2.6926, + "theoretical_loss": 3.317334399541649, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008460222412318222, + "loss": 2.8351, + "theoretical_loss": 3.3173234349799166, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008455945252352438, + "loss": 2.6007, + "theoretical_loss": 3.317312471023135, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008451668092386655, + "loss": 2.7641, + "theoretical_loss": 3.3173015076712447, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008447390932420873, + "loss": 2.7725, + "theoretical_loss": 3.3172905449241865, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1667402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6917831897735596, + "objective/train/theoretical_loss": 3.317285063777451, + "objective/train/tokens_used": 71249376, + "theoretical_loss": 3.317285063777451, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.16, + "learning_rate": 0.000844311377245509, + "loss": 2.6852, + "theoretical_loss": 3.3172795827819, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0008438836612489306, + "loss": 2.6591, + "theoretical_loss": 3.3172686212443274, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008434559452523525, + "loss": 2.74, + "theoretical_loss": 3.317257660311408, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008430282292557742, + "loss": 2.83, + "theoretical_loss": 3.3172466999830825, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.17, + "learning_rate": 0.000842600513259196, + "loss": 2.6488, + "theoretical_loss": 3.317235740259292, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008421727972626177, + "loss": 2.5988, + "theoretical_loss": 3.3172247811399767, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008417450812660393, + "loss": 2.6984, + "theoretical_loss": 3.317213822625077, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008413173652694611, + "loss": 2.6926, + "theoretical_loss": 3.3172028647145346, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008408896492728828, + "loss": 2.6579, + "theoretical_loss": 3.317191907408289, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008404619332763046, + "loss": 2.6408, + "theoretical_loss": 3.3171809507062817, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008400342172797262, + "loss": 2.7066, + "theoretical_loss": 3.3171699946084523, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008396065012831479, + "loss": 2.7149, + "theoretical_loss": 3.3171590391147427, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1668521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528013229370117, + "objective/train/theoretical_loss": 3.3171480842250927, + "objective/train/tokens_used": 72887776, + "theoretical_loss": 3.3171480842250927, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008391787852865698, + "loss": 2.6473, + "theoretical_loss": 3.3171480842250927, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008387510692899915, + "loss": 2.6857, + "theoretical_loss": 3.317137129939443, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008383233532934132, + "loss": 2.585, + "theoretical_loss": 3.3171261762577346, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000837895637296835, + "loss": 2.6799, + "theoretical_loss": 3.3171152231799086, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008374679213002566, + "loss": 2.6625, + "theoretical_loss": 3.317104270705905, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008370402053036784, + "loss": 2.6606, + "theoretical_loss": 3.3170933188356644, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008366124893071001, + "loss": 2.7666, + "theoretical_loss": 3.3170823675691277, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008361847733105218, + "loss": 2.7028, + "theoretical_loss": 3.317071416906236, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008357570573139435, + "loss": 2.664, + "theoretical_loss": 3.3170604668469297, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008353293413173653, + "loss": 2.7291, + "theoretical_loss": 3.31704951739115, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008349016253207871, + "loss": 2.6537, + "theoretical_loss": 3.317038568538837, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008344739093242088, + "loss": 2.5645, + "theoretical_loss": 3.317027620289932, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008340461933276305, + "loss": 2.7629, + "theoretical_loss": 3.317016672644375, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1668980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.177739143371582, + "objective/train/theoretical_loss": 3.3170111990478337, + "objective/train/tokens_used": 74526176, + "theoretical_loss": 3.3170111990478337, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0008336184773310522, + "loss": 2.5701, + "theoretical_loss": 3.3170057256021077, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008331907613344739, + "loss": 2.6897, + "theoretical_loss": 3.3169947791630703, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008327630453378957, + "loss": 2.7863, + "theoretical_loss": 3.3169838333272037, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008323353293413174, + "loss": 2.6692, + "theoretical_loss": 3.316972888094449, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.18, + "learning_rate": 0.000831907613344739, + "loss": 2.6618, + "theoretical_loss": 3.3169619434647464, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008314798973481608, + "loss": 2.5543, + "theoretical_loss": 3.3169509994380375, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008310521813515826, + "loss": 2.7893, + "theoretical_loss": 3.3169400560142623, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008306244653550043, + "loss": 2.6742, + "theoretical_loss": 3.3169291131933623, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008301967493584261, + "loss": 2.6488, + "theoretical_loss": 3.316918170975278, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008297690333618477, + "loss": 2.5798, + "theoretical_loss": 3.31690722935995, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008293413173652695, + "loss": 2.7802, + "theoretical_loss": 3.3168962883473205, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008289136013686912, + "loss": 2.7032, + "theoretical_loss": 3.316885347937329, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1670028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.408548355102539, + "objective/train/theoretical_loss": 3.316874408129916, + "objective/train/tokens_used": 76164576, + "theoretical_loss": 3.316874408129916, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008284858853721129, + "loss": 2.7009, + "theoretical_loss": 3.316874408129916, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008280581693755347, + "loss": 2.6379, + "theoretical_loss": 3.316863468925024, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008276304533789563, + "loss": 2.7495, + "theoretical_loss": 3.3168525303225924, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008272027373823782, + "loss": 2.8538, + "theoretical_loss": 3.316841592322563, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008267750213857999, + "loss": 2.8283, + "theoretical_loss": 3.3168306549248765, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008263473053892216, + "loss": 2.6814, + "theoretical_loss": 3.316819718129474, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008259195893926434, + "loss": 2.6987, + "theoretical_loss": 3.3168087819362957, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.18, + "learning_rate": 0.000825491873396065, + "loss": 2.6017, + "theoretical_loss": 3.316797846345283, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008250641573994867, + "loss": 2.7868, + "theoretical_loss": 3.316786911356377, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008246364414029085, + "loss": 2.5386, + "theoretical_loss": 3.316775976969519, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008242087254063302, + "loss": 2.806, + "theoretical_loss": 3.316765043184649, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0008237810094097519, + "loss": 2.7629, + "theoretical_loss": 3.316754110001708, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008233532934131736, + "loss": 2.7679, + "theoretical_loss": 3.3167431774206384, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1670628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.13972806930542, + "objective/train/theoretical_loss": 3.316737711355786, + "objective/train/tokens_used": 77802976, + "theoretical_loss": 3.316737711355786, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008229255774165954, + "loss": 2.5883, + "theoretical_loss": 3.3167322454413792, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008224978614200172, + "loss": 2.7089, + "theoretical_loss": 3.316721314063873, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008220701454234389, + "loss": 2.7955, + "theoretical_loss": 3.3167103832880604, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008216424294268606, + "loss": 2.7126, + "theoretical_loss": 3.316699453113882, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008212147134302823, + "loss": 2.6486, + "theoretical_loss": 3.3166885235412784, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.19, + "learning_rate": 0.000820786997433704, + "loss": 2.6465, + "theoretical_loss": 3.316677594570192, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008203592814371258, + "loss": 2.6168, + "theoretical_loss": 3.316666666200563, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008199315654405474, + "loss": 2.6939, + "theoretical_loss": 3.316655738432332, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008195038494439692, + "loss": 2.6707, + "theoretical_loss": 3.3166448112654408, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.19, + "learning_rate": 0.000819076133447391, + "loss": 2.6281, + "theoretical_loss": 3.3166338846998302, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008186484174508127, + "loss": 2.7623, + "theoretical_loss": 3.316622958735442, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008182207014542345, + "loss": 2.7285, + "theoretical_loss": 3.3166120333722158, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1671752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2737972736358643, + "objective/train/theoretical_loss": 3.3166011086100937, + "objective/train/tokens_used": 79441376, + "theoretical_loss": 3.3166011086100937, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008177929854576561, + "loss": 2.7943, + "theoretical_loss": 3.3166011086100937, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008173652694610778, + "loss": 2.6782, + "theoretical_loss": 3.3165901844490167, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008169375534644996, + "loss": 2.6773, + "theoretical_loss": 3.3165792608889255, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008165098374679213, + "loss": 2.7108, + "theoretical_loss": 3.3165683379297612, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008160821214713431, + "loss": 2.644, + "theoretical_loss": 3.3165574155714657, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008156544054747647, + "loss": 2.7035, + "theoretical_loss": 3.3165464938139797, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008152266894781864, + "loss": 2.6701, + "theoretical_loss": 3.3165355726572434, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008147989734816083, + "loss": 2.6013, + "theoretical_loss": 3.3165246521011995, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00081437125748503, + "loss": 2.5665, + "theoretical_loss": 3.3165137321457885, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008139435414884518, + "loss": 2.5955, + "theoretical_loss": 3.3165028127909513, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0008135158254918734, + "loss": 2.7426, + "theoretical_loss": 3.3164918940366293, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008130881094952951, + "loss": 2.6797, + "theoretical_loss": 3.3164809758827634, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008126603934987169, + "loss": 2.7291, + "theoretical_loss": 3.3164700583292954, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1672176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4254088401794434, + "objective/train/theoretical_loss": 3.316464599777692, + "objective/train/tokens_used": 81079776, + "theoretical_loss": 3.316464599777692, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008122326775021386, + "loss": 2.6885, + "theoretical_loss": 3.316459141376166, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008118049615055603, + "loss": 2.6706, + "theoretical_loss": 3.3164482250233163, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.2, + "learning_rate": 0.000811377245508982, + "loss": 2.6934, + "theoretical_loss": 3.316437309270688, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008109495295124037, + "loss": 2.6764, + "theoretical_loss": 3.316426394118222, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008105218135158256, + "loss": 2.6293, + "theoretical_loss": 3.316415479565859, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008100940975192473, + "loss": 2.7261, + "theoretical_loss": 3.3164045656135417, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008096663815226689, + "loss": 2.6812, + "theoretical_loss": 3.3163936522612096, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008092386655260907, + "loss": 2.5557, + "theoretical_loss": 3.3163827395088052, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008088109495295124, + "loss": 2.6232, + "theoretical_loss": 3.3163718273562695, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008083832335329342, + "loss": 2.5755, + "theoretical_loss": 3.3163609158035436, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008079555175363559, + "loss": 2.5787, + "theoretical_loss": 3.3163500048505687, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008075278015397775, + "loss": 2.6212, + "theoretical_loss": 3.3163390944972857, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1673386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2504236698150635, + "objective/train/theoretical_loss": 3.316328184743637, + "objective/train/tokens_used": 82718176, + "theoretical_loss": 3.316328184743637, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008071000855431993, + "loss": 2.5091, + "theoretical_loss": 3.316328184743637, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008066723695466211, + "loss": 2.7788, + "theoretical_loss": 3.3163172755895634, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008062446535500429, + "loss": 2.6996, + "theoretical_loss": 3.3163063670350055, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008058169375534645, + "loss": 2.639, + "theoretical_loss": 3.3162954590799054, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008053892215568862, + "loss": 2.7192, + "theoretical_loss": 3.316284551724204, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000804961505560308, + "loss": 2.5435, + "theoretical_loss": 3.3162736449678434, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008045337895637297, + "loss": 2.7468, + "theoretical_loss": 3.3162627388107637, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008041060735671514, + "loss": 2.6621, + "theoretical_loss": 3.316251833252908, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008036783575705731, + "loss": 2.6691, + "theoretical_loss": 3.3162409282942154, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0008032506415739948, + "loss": 2.6699, + "theoretical_loss": 3.316230023934629, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0008028229255774167, + "loss": 2.711, + "theoretical_loss": 3.3162191201740896, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0008023952095808384, + "loss": 2.6899, + "theoretical_loss": 3.3162082170125387, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00080196749358426, + "loss": 2.4566, + "theoretical_loss": 3.3161973144499175, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1673816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.680955171585083, + "objective/train/theoretical_loss": 3.316191863393187, + "objective/train/tokens_used": 84356576, + "theoretical_loss": 3.316191863393187, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0008015397775876818, + "loss": 2.5931, + "theoretical_loss": 3.3161864124861675, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0008011120615911035, + "loss": 2.6795, + "theoretical_loss": 3.31617551112123, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0008006843455945253, + "loss": 2.7542, + "theoretical_loss": 3.316164610355047, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.21, + "learning_rate": 0.000800256629597947, + "loss": 2.6446, + "theoretical_loss": 3.316153710187559, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007998289136013686, + "loss": 2.7551, + "theoretical_loss": 3.316142810618708, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007994011976047904, + "loss": 2.7207, + "theoretical_loss": 3.3161319116484353, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007989734816082121, + "loss": 2.8592, + "theoretical_loss": 3.3161210132766823, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.21, + "learning_rate": 0.000798545765611634, + "loss": 2.8129, + "theoretical_loss": 3.316110115503391, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007981180496150557, + "loss": 2.6611, + "theoretical_loss": 3.316099218328502, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007976903336184773, + "loss": 2.7847, + "theoretical_loss": 3.3160883217519572, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007972626176218991, + "loss": 2.5896, + "theoretical_loss": 3.316077425773698, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007968349016253208, + "loss": 2.699, + "theoretical_loss": 3.316066530393666, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1674780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0617194175720215, + "objective/train/theoretical_loss": 3.3160556356118027, + "objective/train/tokens_used": 85994976, + "theoretical_loss": 3.3160556356118027, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007964071856287425, + "loss": 2.7631, + "theoretical_loss": 3.3160556356118027, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007959794696321643, + "loss": 2.8357, + "theoretical_loss": 3.3160447414280494, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007955517536355859, + "loss": 2.6724, + "theoretical_loss": 3.316033847842348, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007951240376390077, + "loss": 2.6585, + "theoretical_loss": 3.3160229548546396, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007946963216424295, + "loss": 2.6504, + "theoretical_loss": 3.316012062464866, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007942686056458512, + "loss": 2.7904, + "theoretical_loss": 3.316001170672968, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.21, + "learning_rate": 0.000793840889649273, + "loss": 2.6297, + "theoretical_loss": 3.3159902794788887, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0007934131736526946, + "loss": 2.6602, + "theoretical_loss": 3.3159793888825684, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007929854576561164, + "loss": 2.7146, + "theoretical_loss": 3.315968498883949, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007925577416595381, + "loss": 2.681, + "theoretical_loss": 3.3159576094829726, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007921300256629598, + "loss": 2.8578, + "theoretical_loss": 3.3159467206795794, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007917023096663815, + "loss": 2.6611, + "theoretical_loss": 3.3159358324737123, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007912745936698032, + "loss": 2.5705, + "theoretical_loss": 3.3159249448653125, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1675938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6079812049865723, + "objective/train/theoretical_loss": 3.3159195012851446, + "objective/train/tokens_used": 87633376, + "theoretical_loss": 3.3159195012851446, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007908468776732249, + "loss": 2.6312, + "theoretical_loss": 3.3159140578543216, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007904191616766468, + "loss": 2.6494, + "theoretical_loss": 3.315903171440681, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007899914456800685, + "loss": 2.5289, + "theoretical_loss": 3.3158922856243325, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007895637296834902, + "loss": 2.6143, + "theoretical_loss": 3.3158814004052175, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007891360136869119, + "loss": 2.6765, + "theoretical_loss": 3.3158705157832786, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007887082976903336, + "loss": 2.6434, + "theoretical_loss": 3.315859631758456, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007882805816937554, + "loss": 2.7441, + "theoretical_loss": 3.3158487483306924, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.22, + "learning_rate": 0.000787852865697177, + "loss": 2.5689, + "theoretical_loss": 3.3158378654999288, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007874251497005988, + "loss": 2.67, + "theoretical_loss": 3.315826983266107, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007869974337040205, + "loss": 2.746, + "theoretical_loss": 3.31581610162917, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007865697177074422, + "loss": 2.6029, + "theoretical_loss": 3.3158052205890574, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007861420017108641, + "loss": 2.5993, + "theoretical_loss": 3.315794340145712, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1676633, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0798161029815674, + "objective/train/theoretical_loss": 3.3157834602990754, + "objective/train/tokens_used": 89271776, + "theoretical_loss": 3.3157834602990754, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007857142857142857, + "loss": 2.6821, + "theoretical_loss": 3.3157834602990754, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007852865697177075, + "loss": 2.6591, + "theoretical_loss": 3.3157725810490892, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007848588537211292, + "loss": 2.5473, + "theoretical_loss": 3.3157617023956956, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007844311377245509, + "loss": 2.737, + "theoretical_loss": 3.3157508243388354, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007840034217279727, + "loss": 2.5373, + "theoretical_loss": 3.315739946878451, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0007835757057313943, + "loss": 2.7326, + "theoretical_loss": 3.3157290700144837, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.22, + "learning_rate": 0.000783147989734816, + "loss": 2.8193, + "theoretical_loss": 3.315718193746876, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007827202737382378, + "loss": 2.8582, + "theoretical_loss": 3.315707318075569, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007822925577416596, + "loss": 2.6654, + "theoretical_loss": 3.3156964430005047, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007818648417450814, + "loss": 2.7077, + "theoretical_loss": 3.315685568521625, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000781437125748503, + "loss": 2.761, + "theoretical_loss": 3.3156746946388713, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007810094097519247, + "loss": 2.6981, + "theoretical_loss": 3.315663821352186, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007805816937553465, + "loss": 2.8834, + "theoretical_loss": 3.3156529486615103, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1677094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.535158634185791, + "objective/train/theoretical_loss": 3.315647512539658, + "objective/train/tokens_used": 90910176, + "theoretical_loss": 3.315647512539658, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007801539777587682, + "loss": 2.7827, + "theoretical_loss": 3.3156420765667862, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00077972626176219, + "loss": 2.7269, + "theoretical_loss": 3.3156312050679553, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007792985457656116, + "loss": 2.751, + "theoretical_loss": 3.31562033416496, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007788708297690333, + "loss": 2.7144, + "theoretical_loss": 3.315609463857742, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007784431137724552, + "loss": 2.6884, + "theoretical_loss": 3.3155985941462425, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007780153977758769, + "loss": 2.6604, + "theoretical_loss": 3.315587725030404, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007775876817792986, + "loss": 2.7039, + "theoretical_loss": 3.315576856510168, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007771599657827203, + "loss": 2.6965, + "theoretical_loss": 3.315565988585477, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.23, + "learning_rate": 0.000776732249786142, + "loss": 2.7133, + "theoretical_loss": 3.3155551212562724, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007763045337895638, + "loss": 2.8031, + "theoretical_loss": 3.3155442545224956, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007758768177929855, + "loss": 2.6672, + "theoretical_loss": 3.315533388384089, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007754491017964071, + "loss": 2.7976, + "theoretical_loss": 3.315522522840995, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1678209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4061381816864014, + "objective/train/theoretical_loss": 3.3155116578931545, + "objective/train/tokens_used": 92548576, + "theoretical_loss": 3.3155116578931545, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007750213857998289, + "loss": 2.4624, + "theoretical_loss": 3.3155116578931545, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007745936698032506, + "loss": 2.7517, + "theoretical_loss": 3.31550079354051, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007741659538066725, + "loss": 2.7074, + "theoretical_loss": 3.315489929783004, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007737382378100942, + "loss": 2.7777, + "theoretical_loss": 3.315479066620577, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007733105218135158, + "loss": 2.7602, + "theoretical_loss": 3.3154682040531718, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0007728828058169376, + "loss": 2.6515, + "theoretical_loss": 3.3154573420807303, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007724550898203593, + "loss": 2.7385, + "theoretical_loss": 3.3154464807031943, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007720273738237811, + "loss": 2.6795, + "theoretical_loss": 3.315435619920506, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007715996578272027, + "loss": 2.6428, + "theoretical_loss": 3.315424759732607, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007711719418306244, + "loss": 2.6636, + "theoretical_loss": 3.31541390013944, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007707442258340462, + "loss": 2.6116, + "theoretical_loss": 3.3154030411409465, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007703165098374679, + "loss": 2.6947, + "theoretical_loss": 3.3153921827370683, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007698887938408897, + "loss": 2.6784, + "theoretical_loss": 3.3153813249277473, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1678783, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.995398759841919, + "objective/train/theoretical_loss": 3.315375896246028, + "objective/train/tokens_used": 94186976, + "theoretical_loss": 3.315375896246028, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007694610778443114, + "loss": 2.712, + "theoretical_loss": 3.3153704677129263, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007690333618477331, + "loss": 2.6632, + "theoretical_loss": 3.3153596110925467, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007686056458511549, + "loss": 2.6325, + "theoretical_loss": 3.315348755066551, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007681779298545766, + "loss": 2.611, + "theoretical_loss": 3.31533789963488, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007677502138579982, + "loss": 2.5939, + "theoretical_loss": 3.3153270447974776, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00076732249786142, + "loss": 2.7007, + "theoretical_loss": 3.3153161905542845, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007668947818648417, + "loss": 2.6422, + "theoretical_loss": 3.315305336905243, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007664670658682635, + "loss": 2.6363, + "theoretical_loss": 3.3152944838502956, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007660393498716853, + "loss": 2.7748, + "theoretical_loss": 3.3152836313893843, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007656116338751069, + "loss": 2.8551, + "theoretical_loss": 3.3152727795224504, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007651839178785287, + "loss": 2.6915, + "theoretical_loss": 3.3152619282494373, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007647562018819504, + "loss": 2.4881, + "theoretical_loss": 3.315251077570286, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1679875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7984094619750977, + "objective/train/theoretical_loss": 3.3152402274849395, + "objective/train/tokens_used": 95825376, + "theoretical_loss": 3.3152402274849395, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007643284858853722, + "loss": 2.7687, + "theoretical_loss": 3.3152402274849395, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007639007698887939, + "loss": 2.5306, + "theoretical_loss": 3.315229377993339, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007634730538922155, + "loss": 2.5974, + "theoretical_loss": 3.315218529095427, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0007630453378956373, + "loss": 2.6238, + "theoretical_loss": 3.315207680791146, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.25, + "learning_rate": 0.000762617621899059, + "loss": 2.6609, + "theoretical_loss": 3.3151968330804378, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007621899059024807, + "loss": 2.6483, + "theoretical_loss": 3.3151859859632444, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007617621899059026, + "loss": 2.5367, + "theoretical_loss": 3.3151751394395084, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007613344739093242, + "loss": 2.8444, + "theoretical_loss": 3.3151642935091714, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.25, + "learning_rate": 0.000760906757912746, + "loss": 2.5866, + "theoretical_loss": 3.3151534481721763, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007604790419161677, + "loss": 2.7557, + "theoretical_loss": 3.3151426034284643, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007600513259195894, + "loss": 2.8116, + "theoretical_loss": 3.3151317592779788, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007596236099230111, + "loss": 2.7216, + "theoretical_loss": 3.315120915720661, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007591958939264328, + "loss": 2.6847, + "theoretical_loss": 3.315110072756454, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1680552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6995866298675537, + "objective/train/theoretical_loss": 3.3151046514967484, + "objective/train/tokens_used": 97463776, + "theoretical_loss": 3.3151046514967484, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007587681779298546, + "loss": 2.5473, + "theoretical_loss": 3.315099230385299, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007583404619332763, + "loss": 2.6589, + "theoretical_loss": 3.3150883886071387, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007579127459366981, + "loss": 2.5557, + "theoretical_loss": 3.3150775474219154, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007574850299401198, + "loss": 2.6966, + "theoretical_loss": 3.3150667068295716, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007570573139435415, + "loss": 2.6355, + "theoretical_loss": 3.3150558668300487, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007566295979469632, + "loss": 2.6891, + "theoretical_loss": 3.31504502742329, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.25, + "learning_rate": 0.000756201881950385, + "loss": 2.6008, + "theoretical_loss": 3.3150341886092374, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007557741659538067, + "loss": 2.6525, + "theoretical_loss": 3.3150233503878326, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007553464499572284, + "loss": 2.7266, + "theoretical_loss": 3.3150125127590186, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007549187339606501, + "loss": 2.6693, + "theoretical_loss": 3.3150016757227374, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007544910179640718, + "loss": 2.5191, + "theoretical_loss": 3.314990839278931, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007540633019674937, + "loss": 2.5744, + "theoretical_loss": 3.314980003427542, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1681818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.981553554534912, + "objective/train/theoretical_loss": 3.3149691681685134, + "objective/train/tokens_used": 99102176, + "theoretical_loss": 3.3149691681685134, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007536355859709153, + "loss": 2.6833, + "theoretical_loss": 3.3149691681685134, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007532078699743371, + "loss": 2.7529, + "theoretical_loss": 3.314958333501786, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0007527801539777588, + "loss": 2.5656, + "theoretical_loss": 3.3149474994273036, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007523524379811805, + "loss": 2.7531, + "theoretical_loss": 3.314936665945008, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007519247219846023, + "loss": 2.6828, + "theoretical_loss": 3.314925833054841, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007514970059880239, + "loss": 2.6544, + "theoretical_loss": 3.3149150007567454, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007510692899914457, + "loss": 2.6764, + "theoretical_loss": 3.314904169050664, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007506415739948674, + "loss": 2.6181, + "theoretical_loss": 3.3148933379365384, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007502138579982891, + "loss": 2.5999, + "theoretical_loss": 3.3148825074143113, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.26, + "learning_rate": 0.000749786142001711, + "loss": 2.5656, + "theoretical_loss": 3.314871677483925, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007493584260051326, + "loss": 2.6492, + "theoretical_loss": 3.3148608481453223, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007489307100085543, + "loss": 2.6454, + "theoretical_loss": 3.3148500193984454, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007485029940119761, + "loss": 2.6889, + "theoretical_loss": 3.3148391912432364, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1682472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.804523468017578, + "objective/train/theoretical_loss": 3.3148337773874896, + "objective/train/tokens_used": 100740576, + "theoretical_loss": 3.3148337773874896, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007480752780153978, + "loss": 2.6604, + "theoretical_loss": 3.3148283636796383, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007476475620188196, + "loss": 2.6386, + "theoretical_loss": 3.3148175367075927, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007472198460222412, + "loss": 2.6414, + "theoretical_loss": 3.314806710327043, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007467921300256629, + "loss": 2.8612, + "theoretical_loss": 3.3147958845379306, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007463644140290847, + "loss": 2.5636, + "theoretical_loss": 3.3147850593401986, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007459366980325064, + "loss": 2.7602, + "theoretical_loss": 3.3147742347337896, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007455089820359282, + "loss": 2.8901, + "theoretical_loss": 3.3147634107186454, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007450812660393499, + "loss": 2.6674, + "theoretical_loss": 3.3147525872947092, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007446535500427716, + "loss": 2.5025, + "theoretical_loss": 3.3147417644619233, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007442258340461934, + "loss": 2.6975, + "theoretical_loss": 3.31473094222023, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007437981180496151, + "loss": 2.8009, + "theoretical_loss": 3.3147201205695715, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007433704020530368, + "loss": 2.5947, + "theoretical_loss": 3.3147092995098912, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1683394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7559075355529785, + "objective/train/theoretical_loss": 3.3146984790411307, + "objective/train/tokens_used": 102378976, + "theoretical_loss": 3.3146984790411307, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0007429426860564585, + "loss": 2.6029, + "theoretical_loss": 3.3146984790411307, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007425149700598802, + "loss": 2.7892, + "theoretical_loss": 3.314687659163233, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.27, + "learning_rate": 0.000742087254063302, + "loss": 2.4519, + "theoretical_loss": 3.3146768398761406, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007416595380667238, + "loss": 2.7062, + "theoretical_loss": 3.3146660211797956, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007412318220701454, + "loss": 2.6081, + "theoretical_loss": 3.3146552030741416, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007408041060735672, + "loss": 2.6677, + "theoretical_loss": 3.31464438555912, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007403763900769889, + "loss": 2.5899, + "theoretical_loss": 3.314633568634674, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007399486740804107, + "loss": 2.7511, + "theoretical_loss": 3.314622752300746, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007395209580838323, + "loss": 2.6366, + "theoretical_loss": 3.3146119365572786, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000739093242087254, + "loss": 2.5128, + "theoretical_loss": 3.3146011214042144, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007386655260906758, + "loss": 2.6395, + "theoretical_loss": 3.314590306841496, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007382378100940975, + "loss": 2.808, + "theoretical_loss": 3.3145794928690657, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007378100940975194, + "loss": 2.7223, + "theoretical_loss": 3.3145686794868667, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1684169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4948744773864746, + "objective/train/theoretical_loss": 3.314563273017086, + "objective/train/tokens_used": 104017376, + "theoretical_loss": 3.314563273017086, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.27, + "learning_rate": 0.000737382378100941, + "loss": 2.7059, + "theoretical_loss": 3.314557866694841, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007369546621043627, + "loss": 2.8122, + "theoretical_loss": 3.314547054492932, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007365269461077845, + "loss": 2.7304, + "theoretical_loss": 3.314536242881082, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007360992301112062, + "loss": 2.7804, + "theoretical_loss": 3.3145254318592325, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007356715141146278, + "loss": 2.6001, + "theoretical_loss": 3.3145146214273282, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007352437981180496, + "loss": 2.6449, + "theoretical_loss": 3.3145038115853103, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007348160821214713, + "loss": 2.6983, + "theoretical_loss": 3.314493002333122, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007343883661248931, + "loss": 2.7784, + "theoretical_loss": 3.3144821936707056, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007339606501283148, + "loss": 2.6079, + "theoretical_loss": 3.3144713855980044, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007335329341317365, + "loss": 2.6211, + "theoretical_loss": 3.3144605781149608, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0007331052181351583, + "loss": 2.6204, + "theoretical_loss": 3.314449771221517, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00073267750213858, + "loss": 2.7189, + "theoretical_loss": 3.3144389649176165, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1684867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.818236827850342, + "objective/train/theoretical_loss": 3.3144281592032017, + "objective/train/tokens_used": 105655776, + "theoretical_loss": 3.3144281592032017, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007322497861420018, + "loss": 2.7713, + "theoretical_loss": 3.3144281592032017, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007318220701454235, + "loss": 2.5885, + "theoretical_loss": 3.3144173540782154, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007313943541488451, + "loss": 2.6047, + "theoretical_loss": 3.3144065495426, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007309666381522669, + "loss": 2.6418, + "theoretical_loss": 3.3143957455962982, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007305389221556886, + "loss": 2.7568, + "theoretical_loss": 3.3143849422392533, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007301112061591104, + "loss": 2.6375, + "theoretical_loss": 3.3143741394714077, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007296834901625322, + "loss": 2.761, + "theoretical_loss": 3.314363337292704, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007292557741659538, + "loss": 2.731, + "theoretical_loss": 3.314352535703086, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007288280581693756, + "loss": 2.7631, + "theoretical_loss": 3.314341734702495, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007284003421727973, + "loss": 2.6186, + "theoretical_loss": 3.3143309342908744, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.28, + "learning_rate": 0.000727972626176219, + "loss": 2.6961, + "theoretical_loss": 3.314320134468167, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007275449101796407, + "loss": 2.7757, + "theoretical_loss": 3.3143093352343165, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007271171941830624, + "loss": 2.6504, + "theoretical_loss": 3.314298536589264, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1686119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.809108018875122, + "objective/train/theoretical_loss": 3.3142931374875197, + "objective/train/tokens_used": 107294176, + "theoretical_loss": 3.3142931374875197, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007266894781864842, + "loss": 2.6963, + "theoretical_loss": 3.3142877385329537, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007262617621899059, + "loss": 2.6956, + "theoretical_loss": 3.314276941065328, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007258340461933276, + "loss": 2.7936, + "theoretical_loss": 3.3142661441863295, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007254063301967494, + "loss": 2.6945, + "theoretical_loss": 3.3142553478959007, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007249786142001711, + "loss": 2.634, + "theoretical_loss": 3.3142445521939856, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007245508982035929, + "loss": 2.7157, + "theoretical_loss": 3.314233757080526, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007241231822070146, + "loss": 2.6782, + "theoretical_loss": 3.3142229625554656, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007236954662104363, + "loss": 2.6866, + "theoretical_loss": 3.3142121686187465, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.28, + "learning_rate": 0.000723267750213858, + "loss": 2.7431, + "theoretical_loss": 3.3142013752703123, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007228400342172797, + "loss": 2.6258, + "theoretical_loss": 3.314190582510105, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0007224123182207014, + "loss": 2.6916, + "theoretical_loss": 3.3141797903380685, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007219846022241232, + "loss": 2.6934, + "theoretical_loss": 3.314168998754145, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1686853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6352412700653076, + "objective/train/theoretical_loss": 3.314158207758278, + "objective/train/tokens_used": 108932576, + "theoretical_loss": 3.314158207758278, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007215568862275448, + "loss": 2.8914, + "theoretical_loss": 3.314158207758278, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007211291702309667, + "loss": 2.6968, + "theoretical_loss": 3.3141474173504095, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007207014542343884, + "loss": 2.7933, + "theoretical_loss": 3.3141366275304835, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007202737382378101, + "loss": 2.7595, + "theoretical_loss": 3.3141258382984424, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007198460222412319, + "loss": 2.8446, + "theoretical_loss": 3.3141150496542293, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007194183062446535, + "loss": 2.814, + "theoretical_loss": 3.3141042615977865, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007189905902480753, + "loss": 2.6338, + "theoretical_loss": 3.314093474129058, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000718562874251497, + "loss": 2.7369, + "theoretical_loss": 3.314082687247986, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007181351582549187, + "loss": 2.5899, + "theoretical_loss": 3.314071900954514, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007177074422583405, + "loss": 2.7351, + "theoretical_loss": 3.3140611152485846, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007172797262617622, + "loss": 2.7682, + "theoretical_loss": 3.314050330130141, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.29, + "learning_rate": 0.000716852010265184, + "loss": 2.6207, + "theoretical_loss": 3.314039545599126, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007164242942686057, + "loss": 2.6523, + "theoretical_loss": 3.314028761655483, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1688263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5861594676971436, + "objective/train/theoretical_loss": 3.314023369903908, + "objective/train/tokens_used": 110570976, + "theoretical_loss": 3.314023369903908, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007159965782720274, + "loss": 2.6073, + "theoretical_loss": 3.3140179782991552, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007155688622754492, + "loss": 2.7999, + "theoretical_loss": 3.3140071955300847, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007151411462788708, + "loss": 2.6396, + "theoretical_loss": 3.3139964133482147, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007147134302822925, + "loss": 2.6213, + "theoretical_loss": 3.3139856317534893, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007142857142857143, + "loss": 2.7394, + "theoretical_loss": 3.31397485074585, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.29, + "learning_rate": 0.000713857998289136, + "loss": 2.6453, + "theoretical_loss": 3.3139640703252415, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007134302822925579, + "loss": 2.6476, + "theoretical_loss": 3.3139532904916056, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007130025662959795, + "loss": 2.7387, + "theoretical_loss": 3.3139425112448864, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0007125748502994012, + "loss": 2.6949, + "theoretical_loss": 3.3139317325850257, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.3, + "learning_rate": 0.000712147134302823, + "loss": 2.6522, + "theoretical_loss": 3.3139209545119677, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007117194183062447, + "loss": 2.5179, + "theoretical_loss": 3.313910177025655, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007112917023096664, + "loss": 2.5659, + "theoretical_loss": 3.3138994001260307, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1688735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.452139377593994, + "objective/train/theoretical_loss": 3.313888623813038, + "objective/train/tokens_used": 112209376, + "theoretical_loss": 3.313888623813038, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007108639863130881, + "loss": 2.6064, + "theoretical_loss": 3.313888623813038, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007104362703165098, + "loss": 2.6823, + "theoretical_loss": 3.3138778480866202, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007100085543199316, + "loss": 2.6733, + "theoretical_loss": 3.3138670729467203, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007095808383233532, + "loss": 2.7262, + "theoretical_loss": 3.3138562983932816, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007091531223267751, + "loss": 2.6266, + "theoretical_loss": 3.313845524426247, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007087254063301968, + "loss": 2.7868, + "theoretical_loss": 3.3138347510455595, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007082976903336185, + "loss": 2.6533, + "theoretical_loss": 3.3138239782511625, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007078699743370403, + "loss": 2.8224, + "theoretical_loss": 3.3138132060429992, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007074422583404619, + "loss": 2.6842, + "theoretical_loss": 3.313802434421013, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007070145423438836, + "loss": 2.5755, + "theoretical_loss": 3.313791663385146, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007065868263473054, + "loss": 2.7376, + "theoretical_loss": 3.313780892935343, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007061591103507271, + "loss": 2.7195, + "theoretical_loss": 3.313770123071546, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007057313943541489, + "loss": 2.7146, + "theoretical_loss": 3.313759353793699, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1689720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.289289951324463, + "objective/train/theoretical_loss": 3.313753969374489, + "objective/train/tokens_used": 113847776, + "theoretical_loss": 3.313753969374489, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007053036783575705, + "loss": 2.6381, + "theoretical_loss": 3.3137485851017447, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007048759623609923, + "loss": 2.7474, + "theoretical_loss": 3.3137378169956264, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007044482463644141, + "loss": 2.752, + "theoretical_loss": 3.313727049475287, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007040205303678358, + "loss": 2.4917, + "theoretical_loss": 3.3137162825406707, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007035928143712576, + "loss": 2.5718, + "theoretical_loss": 3.31370551619172, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007031650983746792, + "loss": 2.8449, + "theoretical_loss": 3.313694750428378, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007027373823781009, + "loss": 2.5374, + "theoretical_loss": 3.313683985250589, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0007023096663815227, + "loss": 2.7122, + "theoretical_loss": 3.313673220658295, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0007018819503849444, + "loss": 2.5981, + "theoretical_loss": 3.31366245665144, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.000701454234388366, + "loss": 2.5364, + "theoretical_loss": 3.3136516932299673, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0007010265183917879, + "loss": 2.7192, + "theoretical_loss": 3.3136409303938197, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0007005988023952096, + "loss": 2.683, + "theoretical_loss": 3.313630168142941, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1690994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9632725715637207, + "objective/train/theoretical_loss": 3.3136194064772746, + "objective/train/tokens_used": 115486176, + "theoretical_loss": 3.3136194064772746, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0007001710863986314, + "loss": 2.8662, + "theoretical_loss": 3.3136194064772746, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006997433704020531, + "loss": 2.6794, + "theoretical_loss": 3.3136086453967635, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006993156544054747, + "loss": 2.6724, + "theoretical_loss": 3.313597884901351, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006988879384088965, + "loss": 2.5711, + "theoretical_loss": 3.3135871249909803, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006984602224123182, + "loss": 2.5722, + "theoretical_loss": 3.3135763656655954, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00069803250641574, + "loss": 2.5725, + "theoretical_loss": 3.313565606925139, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006976047904191617, + "loss": 2.4772, + "theoretical_loss": 3.313554848769555, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006971770744225833, + "loss": 2.6553, + "theoretical_loss": 3.313544091198786, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006967493584260052, + "loss": 2.4844, + "theoretical_loss": 3.313533334212776, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006963216424294269, + "loss": 2.7982, + "theoretical_loss": 3.3135225778114683, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006958939264328487, + "loss": 2.6032, + "theoretical_loss": 3.313511821994806, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006954662104362704, + "loss": 2.6065, + "theoretical_loss": 3.313501066762733, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.000695038494439692, + "loss": 2.7262, + "theoretical_loss": 3.3134903121151926, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1691628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5217933654785156, + "objective/train/theoretical_loss": 3.313484935010604, + "objective/train/tokens_used": 117124576, + "theoretical_loss": 3.313484935010604, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006946107784431138, + "loss": 2.6959, + "theoretical_loss": 3.3134795580521277, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006941830624465355, + "loss": 2.8117, + "theoretical_loss": 3.313468804573482, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006937553464499572, + "loss": 2.5203, + "theoretical_loss": 3.313458051679199, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006933276304533789, + "loss": 2.7896, + "theoretical_loss": 3.3134472993692223, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006928999144568007, + "loss": 2.6886, + "theoretical_loss": 3.3134365476434953, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006924721984602225, + "loss": 2.69, + "theoretical_loss": 3.313425796501961, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0006920444824636442, + "loss": 2.7939, + "theoretical_loss": 3.3134150459445633, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006916167664670659, + "loss": 2.8524, + "theoretical_loss": 3.313404295971245, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006911890504704876, + "loss": 2.6725, + "theoretical_loss": 3.313393546581951, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006907613344739093, + "loss": 2.5903, + "theoretical_loss": 3.3133827977766237, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006903336184773311, + "loss": 2.741, + "theoretical_loss": 3.3133720495552064, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006899059024807528, + "loss": 2.6592, + "theoretical_loss": 3.3133613019176433, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1692791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493699073791504, + "objective/train/theoretical_loss": 3.3133505548638778, + "objective/train/tokens_used": 118762976, + "theoretical_loss": 3.3133505548638778, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006894781864841744, + "loss": 2.69, + "theoretical_loss": 3.3133505548638778, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006890504704875963, + "loss": 2.7953, + "theoretical_loss": 3.3133398083938532, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.000688622754491018, + "loss": 2.7766, + "theoretical_loss": 3.313329062507513, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006881950384944397, + "loss": 2.6679, + "theoretical_loss": 3.313318317204801, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006877673224978615, + "loss": 2.5952, + "theoretical_loss": 3.3133075724856598, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006873396065012831, + "loss": 2.6835, + "theoretical_loss": 3.313296828350034, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006869118905047049, + "loss": 2.816, + "theoretical_loss": 3.313286084797867, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006864841745081266, + "loss": 2.6975, + "theoretical_loss": 3.3132753418291023, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006860564585115483, + "loss": 2.7337, + "theoretical_loss": 3.3132645994436833, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006856287425149701, + "loss": 2.797, + "theoretical_loss": 3.313253857641554, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006852010265183917, + "loss": 2.7828, + "theoretical_loss": 3.3132431164226572, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006847733105218136, + "loss": 2.5464, + "theoretical_loss": 3.313232375786937, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006843455945252353, + "loss": 2.6839, + "theoretical_loss": 3.3132216357343367, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1693638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.856311082839966, + "objective/train/theoretical_loss": 3.3132162659266893, + "objective/train/tokens_used": 120401376, + "theoretical_loss": 3.3132162659266893, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.000683917878528657, + "loss": 2.5964, + "theoretical_loss": 3.313210896264801, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006834901625320788, + "loss": 2.8131, + "theoretical_loss": 3.3132001573782714, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006830624465355004, + "loss": 2.7374, + "theoretical_loss": 3.3131894190746936, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006826347305389222, + "loss": 2.6969, + "theoretical_loss": 3.3131786813540107, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0006822070145423439, + "loss": 2.7534, + "theoretical_loss": 3.3131679442161657, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006817792985457656, + "loss": 2.7773, + "theoretical_loss": 3.3131572076611024, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006813515825491873, + "loss": 2.6008, + "theoretical_loss": 3.313146471688765, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.33, + "learning_rate": 0.000680923866552609, + "loss": 2.7016, + "theoretical_loss": 3.3131357362990967, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006804961505560308, + "loss": 2.7661, + "theoretical_loss": 3.3131250014920415, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006800684345594526, + "loss": 2.7664, + "theoretical_loss": 3.313114267267543, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006796407185628743, + "loss": 2.7536, + "theoretical_loss": 3.3131035336255446, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.33, + "learning_rate": 0.000679213002566296, + "loss": 2.5962, + "theoretical_loss": 3.31309280056599, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1694249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.020268201828003, + "objective/train/theoretical_loss": 3.3130820680888236, + "objective/train/tokens_used": 122039776, + "theoretical_loss": 3.3130820680888236, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006787852865697177, + "loss": 2.6657, + "theoretical_loss": 3.3130820680888236, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006783575705731394, + "loss": 2.673, + "theoretical_loss": 3.313071336193988, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006779298545765612, + "loss": 2.8094, + "theoretical_loss": 3.313060604881428, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006775021385799828, + "loss": 2.6326, + "theoretical_loss": 3.313049874151087, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006770744225834046, + "loss": 2.8315, + "theoretical_loss": 3.313039144002908, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006766467065868264, + "loss": 2.7993, + "theoretical_loss": 3.313028414436836, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006762189905902481, + "loss": 2.6893, + "theoretical_loss": 3.313017685452814, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006757912745936699, + "loss": 2.6768, + "theoretical_loss": 3.3130069570507854, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006753635585970915, + "loss": 2.6577, + "theoretical_loss": 3.312996229230695, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006749358426005133, + "loss": 2.7931, + "theoretical_loss": 3.312985501992485, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.33, + "learning_rate": 0.000674508126603935, + "loss": 2.6412, + "theoretical_loss": 3.312974775336101, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006740804106073567, + "loss": 2.5949, + "theoretical_loss": 3.312964049261486, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006736526946107785, + "loss": 2.7894, + "theoretical_loss": 3.3129533237685838, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1695060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9143621921539307, + "objective/train/theoretical_loss": 3.312947961240257, + "objective/train/tokens_used": 123678176, + "theoretical_loss": 3.312947961240257, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006732249786142001, + "loss": 2.8566, + "theoretical_loss": 3.3129425988573376, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006727972626176218, + "loss": 2.668, + "theoretical_loss": 3.3129318745276923, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006723695466210437, + "loss": 2.663, + "theoretical_loss": 3.3129211507795913, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0006719418306244654, + "loss": 2.8598, + "theoretical_loss": 3.312910427612978, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006715141146278872, + "loss": 2.7, + "theoretical_loss": 3.312899705027797, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006710863986313088, + "loss": 2.7927, + "theoretical_loss": 3.312888983023991, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006706586826347305, + "loss": 2.7355, + "theoretical_loss": 3.3128782616015053, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006702309666381523, + "loss": 2.768, + "theoretical_loss": 3.312867540760283, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.34, + "learning_rate": 0.000669803250641574, + "loss": 2.6616, + "theoretical_loss": 3.3128568205002678, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006693755346449957, + "loss": 2.7302, + "theoretical_loss": 3.3128461008214036, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006689478186484174, + "loss": 2.7722, + "theoretical_loss": 3.3128353817236347, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006685201026518392, + "loss": 2.7033, + "theoretical_loss": 3.312824663206905, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1695507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0056416988372803, + "objective/train/theoretical_loss": 3.312813945271158, + "objective/train/tokens_used": 125316576, + "theoretical_loss": 3.312813945271158, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.34, + "learning_rate": 0.000668092386655261, + "loss": 2.6719, + "theoretical_loss": 3.312813945271158, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006676646706586827, + "loss": 2.7762, + "theoretical_loss": 3.3128032279163375, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006672369546621043, + "loss": 2.6626, + "theoretical_loss": 3.312792511142388, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006668092386655261, + "loss": 2.7967, + "theoretical_loss": 3.3127817949492533, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006663815226689478, + "loss": 2.7722, + "theoretical_loss": 3.3127710793368768, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006659538066723696, + "loss": 2.8167, + "theoretical_loss": 3.312760364305203, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006655260906757913, + "loss": 2.781, + "theoretical_loss": 3.3127496498541755, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006650983746792129, + "loss": 2.713, + "theoretical_loss": 3.3127389359837385, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006646706586826347, + "loss": 2.7631, + "theoretical_loss": 3.3127282226938357, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006642429426860565, + "loss": 2.6631, + "theoretical_loss": 3.3127175099844113, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006638152266894783, + "loss": 2.7655, + "theoretical_loss": 3.3127067978554092, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006633875106929, + "loss": 2.6604, + "theoretical_loss": 3.3126960863067736, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006629597946963216, + "loss": 2.7047, + "theoretical_loss": 3.3126853753384484, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1696585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8572521209716797, + "objective/train/theoretical_loss": 3.3126800200718844, + "objective/train/tokens_used": 126954976, + "theoretical_loss": 3.3126800200718844, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006625320786997434, + "loss": 2.7243, + "theoretical_loss": 3.312674664950377, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0006621043627031651, + "loss": 2.8646, + "theoretical_loss": 3.3126639551425043, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006616766467065869, + "loss": 2.6294, + "theoretical_loss": 3.3126532459147735, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006612489307100085, + "loss": 2.7048, + "theoretical_loss": 3.3126425372671293, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006608212147134302, + "loss": 2.7503, + "theoretical_loss": 3.3126318291995154, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006603934987168521, + "loss": 2.8657, + "theoretical_loss": 3.312621121711876, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006599657827202738, + "loss": 2.6333, + "theoretical_loss": 3.312610414804155, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006595380667236955, + "loss": 2.5853, + "theoretical_loss": 3.312599708476297, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006591103507271172, + "loss": 2.6782, + "theoretical_loss": 3.3125890027282447, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006586826347305389, + "loss": 2.6384, + "theoretical_loss": 3.3125782975599436, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006582549187339607, + "loss": 2.7164, + "theoretical_loss": 3.312567592971337, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006578272027373824, + "loss": 2.6183, + "theoretical_loss": 3.3125568889623693, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1697815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5997884273529053, + "objective/train/theoretical_loss": 3.3125461855329847, + "objective/train/tokens_used": 128593376, + "theoretical_loss": 3.3125461855329847, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.35, + "learning_rate": 0.000657399486740804, + "loss": 2.6679, + "theoretical_loss": 3.3125461855329847, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006569717707442258, + "loss": 2.7917, + "theoretical_loss": 3.312535482683127, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006565440547476475, + "loss": 2.7107, + "theoretical_loss": 3.3125247804127405, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006561163387510694, + "loss": 2.6212, + "theoretical_loss": 3.312514078721769, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006556886227544911, + "loss": 2.7163, + "theoretical_loss": 3.312503377610157, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006552609067579127, + "loss": 2.6531, + "theoretical_loss": 3.3124926770778487, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006548331907613345, + "loss": 2.7076, + "theoretical_loss": 3.3124819771247878, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006544054747647562, + "loss": 2.7366, + "theoretical_loss": 3.3124712777509187, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006539777587681779, + "loss": 2.6384, + "theoretical_loss": 3.312460578956186, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006535500427715997, + "loss": 2.7348, + "theoretical_loss": 3.3124498807405325, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006531223267750213, + "loss": 2.6972, + "theoretical_loss": 3.312439183103904, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006526946107784431, + "loss": 2.8168, + "theoretical_loss": 3.312428486046244, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006522668947818649, + "loss": 2.8549, + "theoretical_loss": 3.312417789567496, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1698465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.400505304336548, + "objective/train/theoretical_loss": 3.3124124415451974, + "objective/train/tokens_used": 130231776, + "theoretical_loss": 3.3124124415451974, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0006518391787852866, + "loss": 2.5952, + "theoretical_loss": 3.3124070936676056, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006514114627887084, + "loss": 2.6741, + "theoretical_loss": 3.312396398346516, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00065098374679213, + "loss": 2.7083, + "theoretical_loss": 3.3123857036041717, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006505560307955518, + "loss": 2.7966, + "theoretical_loss": 3.3123750094405167, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006501283147989735, + "loss": 2.5402, + "theoretical_loss": 3.3123643158554956, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006497005988023952, + "loss": 2.7004, + "theoretical_loss": 3.312353622849052, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.36, + "learning_rate": 0.000649272882805817, + "loss": 2.5427, + "theoretical_loss": 3.3123429304211314, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006488451668092386, + "loss": 2.5739, + "theoretical_loss": 3.3123322385716767, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006484174508126605, + "loss": 2.6521, + "theoretical_loss": 3.312321547300633, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006479897348160822, + "loss": 2.6286, + "theoretical_loss": 3.3123108566079438, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006475620188195039, + "loss": 2.7447, + "theoretical_loss": 3.3123001664935545, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006471343028229256, + "loss": 2.6519, + "theoretical_loss": 3.3122894769574085, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1699641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7616679668426514, + "objective/train/theoretical_loss": 3.3122787879994497, + "objective/train/tokens_used": 131870176, + "theoretical_loss": 3.3122787879994497, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006467065868263473, + "loss": 2.7428, + "theoretical_loss": 3.3122787879994497, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.36, + "learning_rate": 0.000646278870829769, + "loss": 2.6251, + "theoretical_loss": 3.3122680996196237, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006458511548331908, + "loss": 2.7687, + "theoretical_loss": 3.312257411817874, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006454234388366125, + "loss": 2.605, + "theoretical_loss": 3.3122467245941447, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006449957228400342, + "loss": 2.6613, + "theoretical_loss": 3.312236037948381, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006445680068434559, + "loss": 2.7783, + "theoretical_loss": 3.312225351880526, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006441402908468777, + "loss": 2.5957, + "theoretical_loss": 3.312214666390525, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006437125748502995, + "loss": 2.6654, + "theoretical_loss": 3.312203981478322, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006432848588537211, + "loss": 2.7803, + "theoretical_loss": 3.3121932971438612, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006428571428571429, + "loss": 2.7261, + "theoretical_loss": 3.3121826133870873, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006424294268605646, + "loss": 2.728, + "theoretical_loss": 3.3121719302079446, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006420017108639863, + "loss": 2.6695, + "theoretical_loss": 3.3121612476063778, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0006415739948674081, + "loss": 2.6719, + "theoretical_loss": 3.3121505655823302, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1699977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.255241870880127, + "objective/train/theoretical_loss": 3.312145224786859, + "objective/train/tokens_used": 133508576, + "theoretical_loss": 3.312145224786859, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006411462788708297, + "loss": 2.6531, + "theoretical_loss": 3.312139884135747, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006407185628742515, + "loss": 2.8272, + "theoretical_loss": 3.3121292032665726, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006402908468776732, + "loss": 2.7125, + "theoretical_loss": 3.3121185229747514, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.37, + "learning_rate": 0.000639863130881095, + "loss": 2.6326, + "theoretical_loss": 3.312107843260227, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006394354148845168, + "loss": 2.6089, + "theoretical_loss": 3.312097164122945, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006390076988879384, + "loss": 2.698, + "theoretical_loss": 3.3120864855628493, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006385799828913601, + "loss": 2.639, + "theoretical_loss": 3.3120758075798844, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006381522668947819, + "loss": 2.7261, + "theoretical_loss": 3.3120651301739947, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006377245508982036, + "loss": 2.7053, + "theoretical_loss": 3.312054453345125, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006372968349016254, + "loss": 2.693, + "theoretical_loss": 3.3120437770932183, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.37, + "learning_rate": 0.000636869118905047, + "loss": 2.7224, + "theoretical_loss": 3.312033101418221, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006364414029084687, + "loss": 2.7439, + "theoretical_loss": 3.3120224263200764, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1701288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8335118293762207, + "objective/train/theoretical_loss": 3.3120117517987295, + "objective/train/tokens_used": 135146976, + "theoretical_loss": 3.3120117517987295, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006360136869118906, + "loss": 2.6846, + "theoretical_loss": 3.3120117517987295, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006355859709153123, + "loss": 2.7079, + "theoretical_loss": 3.312001077854125, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.37, + "learning_rate": 0.000635158254918734, + "loss": 2.6786, + "theoretical_loss": 3.311990404486206, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006347305389221557, + "loss": 2.7235, + "theoretical_loss": 3.311979731694919, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006343028229255774, + "loss": 2.7494, + "theoretical_loss": 3.311969059480207, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006338751069289992, + "loss": 2.7404, + "theoretical_loss": 3.311958387842015, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006334473909324209, + "loss": 2.7239, + "theoretical_loss": 3.311947716780288, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006330196749358425, + "loss": 2.7637, + "theoretical_loss": 3.31193704629497, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006325919589392643, + "loss": 2.739, + "theoretical_loss": 3.311926376386005, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.37, + "learning_rate": 0.000632164242942686, + "loss": 2.7273, + "theoretical_loss": 3.311915707053339, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0006317365269461079, + "loss": 2.7032, + "theoretical_loss": 3.3119050382969153, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006313088109495296, + "loss": 2.5659, + "theoretical_loss": 3.311894370116679, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006308810949529512, + "loss": 2.7242, + "theoretical_loss": 3.3118837025125747, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1702491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.226541519165039, + "objective/train/theoretical_loss": 3.3118783689265547, + "objective/train/tokens_used": 136785376, + "theoretical_loss": 3.3118783689265547, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.38, + "learning_rate": 0.000630453378956373, + "loss": 2.6129, + "theoretical_loss": 3.311873035484547, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006300256629597947, + "loss": 2.5906, + "theoretical_loss": 3.31186236903254, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006295979469632165, + "loss": 2.7031, + "theoretical_loss": 3.311851703156499, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006291702309666381, + "loss": 2.8119, + "theoretical_loss": 3.311841037856368, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006287425149700598, + "loss": 2.5635, + "theoretical_loss": 3.311830373132092, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006283147989734816, + "loss": 2.6324, + "theoretical_loss": 3.3118197089836157, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006278870829769034, + "loss": 2.5702, + "theoretical_loss": 3.3118090454108833, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006274593669803252, + "loss": 2.7085, + "theoretical_loss": 3.3117983824138397, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006270316509837468, + "loss": 2.6944, + "theoretical_loss": 3.311787719992429, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006266039349871685, + "loss": 2.6122, + "theoretical_loss": 3.311777058146597, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006261762189905903, + "loss": 2.5744, + "theoretical_loss": 3.311766396876288, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.38, + "learning_rate": 0.000625748502994012, + "loss": 2.6393, + "theoretical_loss": 3.3117557361814454, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1702889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.460357189178467, + "objective/train/theoretical_loss": 3.3117450760620155, + "objective/train/tokens_used": 138423776, + "theoretical_loss": 3.3117450760620155, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006253207869974336, + "loss": 2.5455, + "theoretical_loss": 3.3117450760620155, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006248930710008554, + "loss": 2.699, + "theoretical_loss": 3.311734416517942, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006244653550042771, + "loss": 2.5352, + "theoretical_loss": 3.31172375754917, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006240376390076989, + "loss": 2.773, + "theoretical_loss": 3.311713099155644, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006236099230111207, + "loss": 2.6753, + "theoretical_loss": 3.311702441337309, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006231822070145423, + "loss": 2.4414, + "theoretical_loss": 3.3116917840941094, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006227544910179641, + "loss": 2.6296, + "theoretical_loss": 3.3116811274259903, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006223267750213858, + "loss": 2.7039, + "theoretical_loss": 3.3116704713328957, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006218990590248076, + "loss": 2.7442, + "theoretical_loss": 3.311659815814771, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0006214713430282293, + "loss": 2.6294, + "theoretical_loss": 3.311649160871561, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006210436270316509, + "loss": 2.6241, + "theoretical_loss": 3.31163850650321, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006206159110350727, + "loss": 2.6799, + "theoretical_loss": 3.311627852709663, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006201881950384944, + "loss": 2.5416, + "theoretical_loss": 3.3116171994908647, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1704007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5959293842315674, + "objective/train/theoretical_loss": 3.311611873096979, + "objective/train/tokens_used": 140062176, + "theoretical_loss": 3.311611873096979, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006197604790419162, + "loss": 2.6409, + "theoretical_loss": 3.3116065468467597, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.39, + "learning_rate": 0.000619332763045338, + "loss": 2.6642, + "theoretical_loss": 3.311595894777293, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006189050470487596, + "loss": 2.584, + "theoretical_loss": 3.3115852432824093, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006184773310521814, + "loss": 2.7318, + "theoretical_loss": 3.311574592362054, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006180496150556031, + "loss": 2.5137, + "theoretical_loss": 3.311563942016171, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006176218990590248, + "loss": 2.7217, + "theoretical_loss": 3.311553292244705, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006171941830624465, + "loss": 2.6947, + "theoretical_loss": 3.3115426430476016, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006167664670658682, + "loss": 2.6158, + "theoretical_loss": 3.3115319944248056, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00061633875106929, + "loss": 2.7412, + "theoretical_loss": 3.311521346376261, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006159110350727117, + "loss": 2.6665, + "theoretical_loss": 3.311510698901914, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006154833190761335, + "loss": 2.6229, + "theoretical_loss": 3.311500052001708, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006150556030795552, + "loss": 2.7601, + "theoretical_loss": 3.311489405675588, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1704747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.608757257461548, + "objective/train/theoretical_loss": 3.3114787599235003, + "objective/train/tokens_used": 141700576, + "theoretical_loss": 3.3114787599235003, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006146278870829769, + "loss": 2.5798, + "theoretical_loss": 3.3114787599235003, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006142001710863987, + "loss": 2.8198, + "theoretical_loss": 3.3114681147453884, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006137724550898204, + "loss": 2.7011, + "theoretical_loss": 3.311457470141198, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.39, + "learning_rate": 0.000613344739093242, + "loss": 2.5222, + "theoretical_loss": 3.311446826110873, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006129170230966638, + "loss": 2.7046, + "theoretical_loss": 3.311436182654359, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006124893071000855, + "loss": 2.532, + "theoretical_loss": 3.311425539771601, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006120615911035072, + "loss": 2.6489, + "theoretical_loss": 3.3114148974625435, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0006116338751069291, + "loss": 2.6925, + "theoretical_loss": 3.3114042557271315, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006112061591103508, + "loss": 2.5338, + "theoretical_loss": 3.3113936145653105, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006107784431137725, + "loss": 2.7481, + "theoretical_loss": 3.3113829739770244, + "tokens_seen": 3112828928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006103507271171942, + "loss": 2.6102, + "theoretical_loss": 3.311372333962219, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006099230111206159, + "loss": 2.6382, + "theoretical_loss": 3.311361694520839, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006094952951240377, + "loss": 2.7085, + "theoretical_loss": 3.311351055652829, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1705767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.969193458557129, + "objective/train/theoretical_loss": 3.311345736433821, + "objective/train/tokens_used": 143338976, + "theoretical_loss": 3.311345736433821, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006090675791274593, + "loss": 2.6841, + "theoretical_loss": 3.3113404173581342, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006086398631308811, + "loss": 2.785, + "theoretical_loss": 3.3113297796367, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006082121471343028, + "loss": 2.4887, + "theoretical_loss": 3.311319142488471, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006077844311377245, + "loss": 2.7748, + "theoretical_loss": 3.311308505913392, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006073567151411464, + "loss": 2.628, + "theoretical_loss": 3.3112978699114084, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.4, + "learning_rate": 0.000606928999144568, + "loss": 2.6668, + "theoretical_loss": 3.3112872344824646, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006065012831479898, + "loss": 2.6537, + "theoretical_loss": 3.3112765996265066, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006060735671514115, + "loss": 2.6849, + "theoretical_loss": 3.3112659653434786, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006056458511548332, + "loss": 2.5396, + "theoretical_loss": 3.3112553316333253, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.4, + "learning_rate": 0.000605218135158255, + "loss": 2.697, + "theoretical_loss": 3.311244698495993, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006047904191616766, + "loss": 2.7287, + "theoretical_loss": 3.3112340659314254, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006043627031650983, + "loss": 2.7988, + "theoretical_loss": 3.3112234339395687, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1706383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63397216796875, + "objective/train/theoretical_loss": 3.311212802520367, + "objective/train/tokens_used": 144977376, + "theoretical_loss": 3.311212802520367, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006039349871685201, + "loss": 2.6021, + "theoretical_loss": 3.311212802520367, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006035072711719419, + "loss": 2.6364, + "theoretical_loss": 3.311202171673766, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006030795551753637, + "loss": 2.6637, + "theoretical_loss": 3.3111915413997104, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006026518391787853, + "loss": 2.7004, + "theoretical_loss": 3.3111809116981457, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.4, + "learning_rate": 0.000602224123182207, + "loss": 2.5709, + "theoretical_loss": 3.3111702825690164, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006017964071856288, + "loss": 2.5791, + "theoretical_loss": 3.311159654012268, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0006013686911890505, + "loss": 2.7283, + "theoretical_loss": 3.311149026027845, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0006009409751924722, + "loss": 2.6204, + "theoretical_loss": 3.3111383986156935, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0006005132591958939, + "loss": 2.7792, + "theoretical_loss": 3.3111277717757583, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0006000855431993156, + "loss": 2.7426, + "theoretical_loss": 3.3111171455079838, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005996578272027374, + "loss": 2.7488, + "theoretical_loss": 3.311106519812316, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005992301112061592, + "loss": 2.7775, + "theoretical_loss": 3.3110958946887, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005988023952095808, + "loss": 2.6377, + "theoretical_loss": 3.31108527013708, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1707531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5469744205474854, + "objective/train/theoretical_loss": 3.3110799580757515, + "objective/train/tokens_used": 146615776, + "theoretical_loss": 3.3110799580757515, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005983746792130026, + "loss": 2.7651, + "theoretical_loss": 3.311074646157402, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005979469632164243, + "loss": 2.7209, + "theoretical_loss": 3.311064022749611, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005975192472198461, + "loss": 2.6821, + "theoretical_loss": 3.311053399913652, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005970915312232677, + "loss": 2.7678, + "theoretical_loss": 3.3110427776494706, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005966638152266894, + "loss": 2.6693, + "theoretical_loss": 3.3110321559570117, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005962360992301112, + "loss": 2.6511, + "theoretical_loss": 3.31102153483622, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005958083832335329, + "loss": 2.7087, + "theoretical_loss": 3.3110109142870416, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005953806672369548, + "loss": 2.6943, + "theoretical_loss": 3.311000294309421, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005949529512403764, + "loss": 2.7683, + "theoretical_loss": 3.310989674903304, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005945252352437981, + "loss": 2.7531, + "theoretical_loss": 3.310979056068635, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005940975192472199, + "loss": 2.4833, + "theoretical_loss": 3.3109684378053603, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005936698032506416, + "loss": 2.67, + "theoretical_loss": 3.3109578201134244, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1708692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8297624588012695, + "objective/train/theoretical_loss": 3.3109472029927725, + "objective/train/tokens_used": 148254176, + "theoretical_loss": 3.3109472029927725, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005932420872540634, + "loss": 2.6339, + "theoretical_loss": 3.3109472029927725, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.41, + "learning_rate": 0.000592814371257485, + "loss": 2.5972, + "theoretical_loss": 3.31093658644335, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005923866552609067, + "loss": 2.642, + "theoretical_loss": 3.3109259704651026, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005919589392643285, + "loss": 2.6567, + "theoretical_loss": 3.3109153550579746, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0005915312232677502, + "loss": 2.7582, + "theoretical_loss": 3.3109047402219125, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.41, + "learning_rate": 0.000591103507271172, + "loss": 2.6444, + "theoretical_loss": 3.3108941259568607, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005906757912745937, + "loss": 2.5592, + "theoretical_loss": 3.3108835122627647, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005902480752780154, + "loss": 2.7232, + "theoretical_loss": 3.3108728991395697, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005898203592814372, + "loss": 2.6259, + "theoretical_loss": 3.3108622865872213, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005893926432848589, + "loss": 2.6995, + "theoretical_loss": 3.3108516746056647, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005889649272882805, + "loss": 2.6163, + "theoretical_loss": 3.310841063194845, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005885372112917023, + "loss": 2.6846, + "theoretical_loss": 3.3108304523547076, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.42, + "learning_rate": 0.000588109495295124, + "loss": 2.5776, + "theoretical_loss": 3.310819842085198, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1709277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672027826309204, + "objective/train/theoretical_loss": 3.3108145371644113, + "objective/train/tokens_used": 149892576, + "theoretical_loss": 3.3108145371644113, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005876817792985458, + "loss": 2.6058, + "theoretical_loss": 3.3108092323862617, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005872540633019676, + "loss": 2.7047, + "theoretical_loss": 3.3107986232578437, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005868263473053892, + "loss": 2.6548, + "theoretical_loss": 3.310788014699889, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.42, + "learning_rate": 0.000586398631308811, + "loss": 2.6675, + "theoretical_loss": 3.3107774067123437, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005859709153122327, + "loss": 2.678, + "theoretical_loss": 3.310766799295153, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005855431993156544, + "loss": 2.8395, + "theoretical_loss": 3.3107561924482622, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005851154833190762, + "loss": 2.5412, + "theoretical_loss": 3.3107455861716164, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005846877673224978, + "loss": 2.6726, + "theoretical_loss": 3.3107349804651616, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005842600513259196, + "loss": 2.6076, + "theoretical_loss": 3.3107243753288427, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005838323353293413, + "loss": 2.7076, + "theoretical_loss": 3.310713770762605, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.42, + "learning_rate": 0.000583404619332763, + "loss": 2.7267, + "theoretical_loss": 3.3107031667663946, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005829769033361848, + "loss": 2.588, + "theoretical_loss": 3.3106925633401563, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1710494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.796520709991455, + "objective/train/theoretical_loss": 3.3106819604838353, + "objective/train/tokens_used": 151530976, + "theoretical_loss": 3.3106819604838353, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005825491873396065, + "loss": 2.7382, + "theoretical_loss": 3.3106819604838353, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005821214713430283, + "loss": 2.5081, + "theoretical_loss": 3.310671358197378, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00058169375534645, + "loss": 2.6232, + "theoretical_loss": 3.3106607564807295, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0005812660393498717, + "loss": 2.6167, + "theoretical_loss": 3.3106501553338346, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005808383233532934, + "loss": 2.6441, + "theoretical_loss": 3.3106395547566394, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005804106073567151, + "loss": 2.7695, + "theoretical_loss": 3.3106289547490895, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005799828913601369, + "loss": 2.6409, + "theoretical_loss": 3.3106183553111297, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005795551753635586, + "loss": 2.734, + "theoretical_loss": 3.310607756442706, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005791274593669804, + "loss": 2.5407, + "theoretical_loss": 3.3105971581437634, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005786997433704021, + "loss": 2.5927, + "theoretical_loss": 3.310586560414248, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005782720273738238, + "loss": 2.6425, + "theoretical_loss": 3.3105759632541054, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005778443113772455, + "loss": 2.5758, + "theoretical_loss": 3.3105653666632806, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005774165953806673, + "loss": 2.5679, + "theoretical_loss": 3.310554770641719, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1711181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.426276683807373, + "objective/train/theoretical_loss": 3.310549472844395, + "objective/train/tokens_used": 153169376, + "theoretical_loss": 3.310549472844395, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005769888793840889, + "loss": 2.5955, + "theoretical_loss": 3.3105441751893667, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005765611633875107, + "loss": 2.6546, + "theoretical_loss": 3.310533580306169, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005761334473909324, + "loss": 2.6795, + "theoretical_loss": 3.310522985992071, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005757057313943541, + "loss": 2.6767, + "theoretical_loss": 3.310512392247019, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005752780153977759, + "loss": 2.7539, + "theoretical_loss": 3.310501799070958, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005748502994011976, + "loss": 2.5585, + "theoretical_loss": 3.310491206463834, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005744225834046194, + "loss": 2.7497, + "theoretical_loss": 3.3104806144255923, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005739948674080411, + "loss": 2.73, + "theoretical_loss": 3.3104700229561783, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005735671514114628, + "loss": 2.7256, + "theoretical_loss": 3.310459432055538, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005731394354148846, + "loss": 2.5896, + "theoretical_loss": 3.3104488417236166, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005727117194183062, + "loss": 2.6611, + "theoretical_loss": 3.31043825196036, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.43, + "learning_rate": 0.000572284003421728, + "loss": 2.5432, + "theoretical_loss": 3.310427662765714, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1712507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5778708457946777, + "objective/train/theoretical_loss": 3.310417074139624, + "objective/train/tokens_used": 154807776, + "theoretical_loss": 3.310417074139624, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005718562874251497, + "loss": 2.4868, + "theoretical_loss": 3.310417074139624, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005714285714285714, + "loss": 2.5066, + "theoretical_loss": 3.3104064860820355, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0005710008554319933, + "loss": 2.7568, + "theoretical_loss": 3.310395898592894, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005705731394354149, + "loss": 2.5419, + "theoretical_loss": 3.310385311672145, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005701454234388366, + "loss": 2.4798, + "theoretical_loss": 3.3103747253197353, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005697177074422584, + "loss": 2.7052, + "theoretical_loss": 3.310364139535609, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005692899914456801, + "loss": 2.5732, + "theoretical_loss": 3.3103535543197133, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005688622754491018, + "loss": 2.8066, + "theoretical_loss": 3.310342969671993, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005684345594525235, + "loss": 2.6306, + "theoretical_loss": 3.3103323855923934, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005680068434559452, + "loss": 2.6615, + "theoretical_loss": 3.3103218020808605, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.44, + "learning_rate": 0.000567579127459367, + "loss": 2.5351, + "theoretical_loss": 3.3103112191373407, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005671514114627887, + "loss": 2.6836, + "theoretical_loss": 3.3103006367617787, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005667236954662105, + "loss": 2.5731, + "theoretical_loss": 3.310290054954121, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1713045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0725793838500977, + "objective/train/theoretical_loss": 3.310284764263239, + "objective/train/tokens_used": 156446176, + "theoretical_loss": 3.310284764263239, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005662959794696322, + "loss": 2.6038, + "theoretical_loss": 3.3102794737143126, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005658682634730539, + "loss": 2.7349, + "theoretical_loss": 3.3102688930423, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005654405474764757, + "loss": 2.6276, + "theoretical_loss": 3.3102583129380285, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005650128314798973, + "loss": 2.5521, + "theoretical_loss": 3.3102477334014435, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.44, + "learning_rate": 0.000564585115483319, + "loss": 2.7029, + "theoretical_loss": 3.3102371544324916, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005641573994867408, + "loss": 2.4412, + "theoretical_loss": 3.3102265760311176, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005637296834901625, + "loss": 2.6061, + "theoretical_loss": 3.310215998197268, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005633019674935843, + "loss": 2.5391, + "theoretical_loss": 3.310205420930888, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.44, + "learning_rate": 0.000562874251497006, + "loss": 2.5292, + "theoretical_loss": 3.310194844231924, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005624465355004277, + "loss": 2.5872, + "theoretical_loss": 3.3101842681003215, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005620188195038495, + "loss": 2.6652, + "theoretical_loss": 3.310173692536026, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005615911035072712, + "loss": 2.7533, + "theoretical_loss": 3.3101631175389836, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1713856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4951908588409424, + "objective/train/theoretical_loss": 3.31015254310914, + "objective/train/tokens_used": 158084576, + "theoretical_loss": 3.31015254310914, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.44, + "learning_rate": 0.000561163387510693, + "loss": 2.6452, + "theoretical_loss": 3.31015254310914, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0005607356715141146, + "loss": 2.561, + "theoretical_loss": 3.310141969246441, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005603079555175363, + "loss": 2.6282, + "theoretical_loss": 3.3101313959508327, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005598802395209581, + "loss": 2.5038, + "theoretical_loss": 3.3101208232222605, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005594525235243798, + "loss": 2.7196, + "theoretical_loss": 3.310110251060671, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005590248075278016, + "loss": 2.7429, + "theoretical_loss": 3.3100996794660085, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005585970915312233, + "loss": 2.6152, + "theoretical_loss": 3.3100891084382207, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.45, + "learning_rate": 0.000558169375534645, + "loss": 2.5593, + "theoretical_loss": 3.310078537977252, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005577416595380668, + "loss": 2.5187, + "theoretical_loss": 3.3100679680830494, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005573139435414885, + "loss": 2.6742, + "theoretical_loss": 3.3100573987555575, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005568862275449101, + "loss": 2.6101, + "theoretical_loss": 3.3100468299947234, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005564585115483319, + "loss": 2.5559, + "theoretical_loss": 3.3100362618004926, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005560307955517536, + "loss": 2.5885, + "theoretical_loss": 3.3100256941728103, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1714507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.492184638977051, + "objective/train/theoretical_loss": 3.3100204105714086, + "objective/train/tokens_used": 159722976, + "theoretical_loss": 3.3100204105714086, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005556030795551754, + "loss": 2.6059, + "theoretical_loss": 3.3100151271116234, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005551753635585971, + "loss": 2.5921, + "theoretical_loss": 3.3100045606168775, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005547476475620188, + "loss": 2.7617, + "theoretical_loss": 3.3099939946885186, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005543199315654406, + "loss": 2.627, + "theoretical_loss": 3.309983429326492, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005538922155688623, + "loss": 2.7206, + "theoretical_loss": 3.309972864530744, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005534644995722841, + "loss": 2.6531, + "theoretical_loss": 3.309962300301221, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005530367835757058, + "loss": 2.5946, + "theoretical_loss": 3.3099517366378683, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005526090675791274, + "loss": 2.6376, + "theoretical_loss": 3.3099411735406323, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005521813515825492, + "loss": 2.558, + "theoretical_loss": 3.3099306110094586, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005517536355859709, + "loss": 2.8317, + "theoretical_loss": 3.3099200490442935, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005513259195893926, + "loss": 2.6789, + "theoretical_loss": 3.3099094876450827, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0005508982035928143, + "loss": 2.7709, + "theoretical_loss": 3.3098989268117722, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1715427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6550986766815186, + "objective/train/theoretical_loss": 3.3098883665443086, + "objective/train/tokens_used": 161361376, + "theoretical_loss": 3.3098883665443086, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005504704875962361, + "loss": 2.6472, + "theoretical_loss": 3.3098883665443086, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005500427715996579, + "loss": 2.6239, + "theoretical_loss": 3.309877806842637, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005496150556030796, + "loss": 2.7941, + "theoretical_loss": 3.3098672477067037, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005491873396065013, + "loss": 2.6612, + "theoretical_loss": 3.309856689136455, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.46, + "learning_rate": 0.000548759623609923, + "loss": 2.6502, + "theoretical_loss": 3.3098461311318363, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005483319076133447, + "loss": 2.7261, + "theoretical_loss": 3.3098355736927947, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005479041916167665, + "loss": 2.4862, + "theoretical_loss": 3.3098250168192753, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005474764756201882, + "loss": 2.7999, + "theoretical_loss": 3.3098144605112245, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005470487596236098, + "loss": 2.6192, + "theoretical_loss": 3.3098039047685885, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005466210436270317, + "loss": 2.7375, + "theoretical_loss": 3.3097933495913128, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005461933276304534, + "loss": 2.7532, + "theoretical_loss": 3.309782794979344, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005457656116338752, + "loss": 2.5138, + "theoretical_loss": 3.309772240932628, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005453378956372969, + "loss": 2.7331, + "theoretical_loss": 3.30976168745111, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1716696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.810262441635132, + "objective/train/theoretical_loss": 3.309756410922285, + "objective/train/tokens_used": 162999776, + "theoretical_loss": 3.309756410922285, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005449101796407185, + "loss": 2.8702, + "theoretical_loss": 3.3097511345347383, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005444824636441403, + "loss": 2.6074, + "theoretical_loss": 3.3097405821834567, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.46, + "learning_rate": 0.000544054747647562, + "loss": 2.6812, + "theoretical_loss": 3.309730030397213, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005436270316509837, + "loss": 2.6631, + "theoretical_loss": 3.309719479175952, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005431993156544055, + "loss": 2.7144, + "theoretical_loss": 3.309708928519621, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005427715996578271, + "loss": 2.7022, + "theoretical_loss": 3.309698378428165, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.46, + "learning_rate": 0.000542343883661249, + "loss": 2.5525, + "theoretical_loss": 3.3096878289015303, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005419161676646707, + "loss": 2.6419, + "theoretical_loss": 3.3096772799396637, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005414884516680924, + "loss": 2.6476, + "theoretical_loss": 3.309666731542511, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005410607356715142, + "loss": 2.6109, + "theoretical_loss": 3.3096561837100187, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.46, + "learning_rate": 0.0005406330196749358, + "loss": 2.6221, + "theoretical_loss": 3.3096456364421325, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005402053036783576, + "loss": 2.6817, + "theoretical_loss": 3.3096350897387983, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1717402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.692416191101074, + "objective/train/theoretical_loss": 3.309624543599963, + "objective/train/tokens_used": 164638176, + "theoretical_loss": 3.309624543599963, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005397775876817793, + "loss": 2.6792, + "theoretical_loss": 3.309624543599963, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.47, + "learning_rate": 0.000539349871685201, + "loss": 2.7542, + "theoretical_loss": 3.3096139980255725, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005389221556886227, + "loss": 2.6407, + "theoretical_loss": 3.3096034530155727, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005384944396920445, + "loss": 2.7596, + "theoretical_loss": 3.3095929085699103, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005380667236954663, + "loss": 2.6674, + "theoretical_loss": 3.309582364688531, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.47, + "learning_rate": 0.000537639007698888, + "loss": 2.7036, + "theoretical_loss": 3.3095718213713816, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005372112917023097, + "loss": 2.6802, + "theoretical_loss": 3.309561278618408, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005367835757057314, + "loss": 2.7301, + "theoretical_loss": 3.309550736429556, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005363558597091531, + "loss": 2.6758, + "theoretical_loss": 3.3095401948047725, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005359281437125748, + "loss": 2.6772, + "theoretical_loss": 3.3095296537440038, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005355004277159966, + "loss": 2.626, + "theoretical_loss": 3.3095191132471955, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005350727117194183, + "loss": 2.5929, + "theoretical_loss": 3.309508573314294, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00053464499572284, + "loss": 2.586, + "theoretical_loss": 3.309498033945246, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1718738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6405951976776123, + "objective/train/theoretical_loss": 3.30949276447215, + "objective/train/tokens_used": 166276576, + "theoretical_loss": 3.30949276447215, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005342172797262618, + "loss": 2.7591, + "theoretical_loss": 3.3094874951399977, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005337895637296835, + "loss": 2.6533, + "theoretical_loss": 3.3094769568984947, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005333618477331053, + "loss": 2.6606, + "theoretical_loss": 3.3094664192206844, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.47, + "learning_rate": 0.000532934131736527, + "loss": 2.523, + "theoretical_loss": 3.309455882106512, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005325064157399487, + "loss": 2.4497, + "theoretical_loss": 3.309445345555925, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005320786997433704, + "loss": 2.6254, + "theoretical_loss": 3.3094348095688684, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005316509837467921, + "loss": 2.4689, + "theoretical_loss": 3.3094242741452895, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005312232677502139, + "loss": 2.631, + "theoretical_loss": 3.3094137392851337, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0005307955517536355, + "loss": 2.6757, + "theoretical_loss": 3.3094032049883486, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005303678357570573, + "loss": 2.4664, + "theoretical_loss": 3.309392671254879, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005299401197604791, + "loss": 2.7248, + "theoretical_loss": 3.309382138084673, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005295124037639008, + "loss": 2.5098, + "theoretical_loss": 3.3093716054776756, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1719408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8767902851104736, + "objective/train/theoretical_loss": 3.3093610734338332, + "objective/train/tokens_used": 167914976, + "theoretical_loss": 3.3093610734338332, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005290846877673226, + "loss": 2.7054, + "theoretical_loss": 3.3093610734338332, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005286569717707442, + "loss": 2.6895, + "theoretical_loss": 3.309350541953093, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005282292557741659, + "loss": 2.5208, + "theoretical_loss": 3.3093400110354008, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005278015397775877, + "loss": 2.6352, + "theoretical_loss": 3.3093294806807028, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005273738237810094, + "loss": 2.4604, + "theoretical_loss": 3.309318950888946, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005269461077844312, + "loss": 2.5577, + "theoretical_loss": 3.3093084216600763, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005265183917878528, + "loss": 2.7499, + "theoretical_loss": 3.3092978929940404, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005260906757912746, + "loss": 2.7004, + "theoretical_loss": 3.309287364890785, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005256629597946964, + "loss": 2.656, + "theoretical_loss": 3.3092768373502555, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005252352437981181, + "loss": 2.5505, + "theoretical_loss": 3.3092663103723994, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005248075278015399, + "loss": 2.6599, + "theoretical_loss": 3.3092557839571626, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005243798118049615, + "loss": 2.5809, + "theoretical_loss": 3.3092452581044913, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005239520958083832, + "loss": 2.6235, + "theoretical_loss": 3.309234732814333, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1720490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.788459300994873, + "objective/train/theoretical_loss": 3.3092294703801786, + "objective/train/tokens_used": 169553376, + "theoretical_loss": 3.3092294703801786, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.48, + "learning_rate": 0.000523524379811805, + "loss": 2.6748, + "theoretical_loss": 3.3092242080866323, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005230966638152267, + "loss": 2.6389, + "theoretical_loss": 3.3092136839213375, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005226689478186483, + "loss": 2.6413, + "theoretical_loss": 3.3092031603183942, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005222412318220702, + "loss": 2.6946, + "theoretical_loss": 3.309192637277749, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005218135158254919, + "loss": 2.6466, + "theoretical_loss": 3.3091821147993485, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005213857998289137, + "loss": 2.7248, + "theoretical_loss": 3.3091715928831396, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0005209580838323354, + "loss": 2.7834, + "theoretical_loss": 3.3091610715290676, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.48, + "learning_rate": 0.000520530367835757, + "loss": 2.6915, + "theoretical_loss": 3.3091505507370798, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005201026518391788, + "loss": 2.5442, + "theoretical_loss": 3.3091400305071232, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005196749358426005, + "loss": 2.6226, + "theoretical_loss": 3.309129510839143, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005192472198460223, + "loss": 2.6848, + "theoretical_loss": 3.309118991733087, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005188195038494439, + "loss": 2.6868, + "theoretical_loss": 3.309108473188901, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1720925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.760850667953491, + "objective/train/theoretical_loss": 3.309097955206532, + "objective/train/tokens_used": 171191776, + "theoretical_loss": 3.309097955206532, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005183917878528656, + "loss": 2.6794, + "theoretical_loss": 3.309097955206532, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005179640718562875, + "loss": 2.7081, + "theoretical_loss": 3.309087437785926, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005175363558597092, + "loss": 2.6129, + "theoretical_loss": 3.30907692092703, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005171086398631309, + "loss": 2.5992, + "theoretical_loss": 3.3090664046297906, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005166809238665526, + "loss": 2.5406, + "theoretical_loss": 3.309055888894154, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005162532078699743, + "loss": 2.6358, + "theoretical_loss": 3.309045373720067, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005158254918733961, + "loss": 2.5887, + "theoretical_loss": 3.3090348591074763, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005153977758768178, + "loss": 2.5329, + "theoretical_loss": 3.3090243450563284, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005149700598802394, + "loss": 2.6037, + "theoretical_loss": 3.30901383156657, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005145423438836612, + "loss": 2.422, + "theoretical_loss": 3.3090033186381476, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.49, + "learning_rate": 0.000514114627887083, + "loss": 2.5692, + "theoretical_loss": 3.3089928062710072, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005136869118905048, + "loss": 2.6477, + "theoretical_loss": 3.3089822944650966, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005132591958939265, + "loss": 2.6837, + "theoretical_loss": 3.308971783220362, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1722164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.840960741043091, + "objective/train/theoretical_loss": 3.3089665278084186, + "objective/train/tokens_used": 172830176, + "theoretical_loss": 3.3089665278084186, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005128314798973481, + "loss": 2.6307, + "theoretical_loss": 3.308961272536749, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005124037639007699, + "loss": 2.5642, + "theoretical_loss": 3.308950762414206, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005119760479041916, + "loss": 2.6576, + "theoretical_loss": 3.3089402528526786, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005115483319076134, + "loss": 2.5886, + "theoretical_loss": 3.3089297438521137, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005111206159110351, + "loss": 2.6957, + "theoretical_loss": 3.308919235412458, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005106928999144567, + "loss": 2.805, + "theoretical_loss": 3.3089087275336575, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0005102651839178785, + "loss": 2.7649, + "theoretical_loss": 3.30889822021566, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005098374679213003, + "loss": 2.6514, + "theoretical_loss": 3.308887713458412, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.5, + "learning_rate": 0.000509409751924722, + "loss": 2.4523, + "theoretical_loss": 3.308877207261859, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005089820359281438, + "loss": 2.6892, + "theoretical_loss": 3.3088667016259494, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005085543199315654, + "loss": 2.6297, + "theoretical_loss": 3.3088561965506287, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005081266039349872, + "loss": 2.7332, + "theoretical_loss": 3.3088456920358436, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1723267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4196202754974365, + "objective/train/theoretical_loss": 3.3088351880815416, + "objective/train/tokens_used": 174468576, + "theoretical_loss": 3.3088351880815416, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005076988879384089, + "loss": 2.5782, + "theoretical_loss": 3.3088351880815416, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005072711719418306, + "loss": 2.5465, + "theoretical_loss": 3.308824684687669, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005068434559452524, + "loss": 2.5736, + "theoretical_loss": 3.308814181854173, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.5, + "learning_rate": 0.000506415739948674, + "loss": 2.5815, + "theoretical_loss": 3.3088036795809996, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005059880239520959, + "loss": 2.6323, + "theoretical_loss": 3.308793177868096, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005055603079555176, + "loss": 2.6393, + "theoretical_loss": 3.308782676715409, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005051325919589393, + "loss": 2.7073, + "theoretical_loss": 3.308772176122885, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.5, + "learning_rate": 0.000504704875962361, + "loss": 2.6812, + "theoretical_loss": 3.3087616760904712, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005042771599657827, + "loss": 2.5786, + "theoretical_loss": 3.308751176618114, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005038494439692045, + "loss": 2.6811, + "theoretical_loss": 3.3087406777057606, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005034217279726262, + "loss": 2.6705, + "theoretical_loss": 3.3087301793533577, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005029940119760479, + "loss": 2.5827, + "theoretical_loss": 3.3087196815608517, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005025662959794696, + "loss": 2.6679, + "theoretical_loss": 3.3087091843281895, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1723860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.738361358642578, + "objective/train/theoretical_loss": 3.3087039359217836, + "objective/train/tokens_used": 176106976, + "theoretical_loss": 3.3087039359217836, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005021385799828913, + "loss": 2.8233, + "theoretical_loss": 3.3086986876553186, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005017108639863131, + "loss": 2.6666, + "theoretical_loss": 3.3086881915421853, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005012831479897349, + "loss": 2.7681, + "theoretical_loss": 3.3086776959887363, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005008554319931566, + "loss": 2.7587, + "theoretical_loss": 3.3086672009949187, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0005004277159965783, + "loss": 2.6563, + "theoretical_loss": 3.3086567065606793, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0005, + "loss": 2.5678, + "theoretical_loss": 3.3086462126859653, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004995722840034218, + "loss": 2.5931, + "theoretical_loss": 3.3086357193707228, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004991445680068435, + "loss": 2.5472, + "theoretical_loss": 3.308625226614899, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004987168520102651, + "loss": 2.5645, + "theoretical_loss": 3.308614734418441, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004982891360136869, + "loss": 2.4611, + "theoretical_loss": 3.3086042427812954, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004978614200171087, + "loss": 2.6291, + "theoretical_loss": 3.3085937517034094, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004974337040205304, + "loss": 2.5455, + "theoretical_loss": 3.30858326118473, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1724798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.940652370452881, + "objective/train/theoretical_loss": 3.308572771225203, + "objective/train/tokens_used": 177745376, + "theoretical_loss": 3.308572771225203, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004970059880239521, + "loss": 2.8201, + "theoretical_loss": 3.308572771225203, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004965782720273738, + "loss": 2.7284, + "theoretical_loss": 3.308562281824777, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004961505560307955, + "loss": 2.6659, + "theoretical_loss": 3.3085517929833976, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004957228400342173, + "loss": 2.6128, + "theoretical_loss": 3.308541304701013, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004952951240376391, + "loss": 2.6806, + "theoretical_loss": 3.3085308169775685, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004948674080410608, + "loss": 2.5461, + "theoretical_loss": 3.308520329813012, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004944396920444824, + "loss": 2.6779, + "theoretical_loss": 3.3085098432072906, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004940119760479042, + "loss": 2.6319, + "theoretical_loss": 3.308499357160351, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.51, + "learning_rate": 0.000493584260051326, + "loss": 2.7657, + "theoretical_loss": 3.3084888716721403, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004931565440547477, + "loss": 2.5879, + "theoretical_loss": 3.308478386742605, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004927288280581693, + "loss": 2.5868, + "theoretical_loss": 3.3084679023716923, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004923011120615911, + "loss": 2.6515, + "theoretical_loss": 3.3084574185593496, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004918733960650129, + "loss": 2.618, + "theoretical_loss": 3.3084469353055237, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1725490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2731151580810547, + "objective/train/theoretical_loss": 3.308441693888038, + "objective/train/tokens_used": 179383776, + "theoretical_loss": 3.308441693888038, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004914456800684346, + "loss": 2.5312, + "theoretical_loss": 3.3084364526101613, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004910179640718563, + "loss": 2.6339, + "theoretical_loss": 3.30842597047321, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.51, + "learning_rate": 0.000490590248075278, + "loss": 2.7708, + "theoretical_loss": 3.3084154888946156, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004901625320786997, + "loss": 2.6738, + "theoretical_loss": 3.3084050078743266, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004897348160821215, + "loss": 2.587, + "theoretical_loss": 3.3083945274122892, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004893071000855432, + "loss": 2.5275, + "theoretical_loss": 3.3083840475084507, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.52, + "learning_rate": 0.000488879384088965, + "loss": 2.5232, + "theoretical_loss": 3.308373568162758, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004884516680923866, + "loss": 2.6663, + "theoretical_loss": 3.3083630893751588, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004880239520958084, + "loss": 2.6747, + "theoretical_loss": 3.3083526111455988, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004875962360992301, + "loss": 2.5472, + "theoretical_loss": 3.308342133474026, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004871685201026518, + "loss": 2.5807, + "theoretical_loss": 3.3083316563603877, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004867408041060736, + "loss": 2.6489, + "theoretical_loss": 3.30832117980463, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1726849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.315721035003662, + "objective/train/theoretical_loss": 3.3083107038067014, + "objective/train/tokens_used": 181022176, + "theoretical_loss": 3.3083107038067014, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048631308810949533, + "loss": 2.6469, + "theoretical_loss": 3.3083107038067014, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048588537211291706, + "loss": 2.5677, + "theoretical_loss": 3.3083002283665475, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048545765611633873, + "loss": 2.599, + "theoretical_loss": 3.3082897534841162, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048502994011976046, + "loss": 2.6397, + "theoretical_loss": 3.308279279159355, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048460222412318225, + "loss": 2.5613, + "theoretical_loss": 3.30826880539221, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.52, + "learning_rate": 0.000484174508126604, + "loss": 2.6644, + "theoretical_loss": 3.308258332182629, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048374679213002565, + "loss": 2.5023, + "theoretical_loss": 3.308247859530559, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004833190761334474, + "loss": 2.6038, + "theoretical_loss": 3.308237387435947, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004828913601368691, + "loss": 2.64, + "theoretical_loss": 3.3082269158987403, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004824636441402909, + "loss": 2.7058, + "theoretical_loss": 3.308216444918886, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004820359281437126, + "loss": 2.5249, + "theoretical_loss": 3.3082059744963317, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004816082121471343, + "loss": 2.5582, + "theoretical_loss": 3.3081955046310236, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.52, + "learning_rate": 0.000481180496150556, + "loss": 2.5963, + "theoretical_loss": 3.3081850353229094, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1727307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.536083221435547, + "objective/train/theoretical_loss": 3.3081798008777836, + "objective/train/tokens_used": 182660576, + "theoretical_loss": 3.3081798008777836, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048075278015397775, + "loss": 2.5723, + "theoretical_loss": 3.3081745665719366, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00048032506415739953, + "loss": 2.5741, + "theoretical_loss": 3.308164098378052, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0004798973481608212, + "loss": 2.6026, + "theoretical_loss": 3.308153630741203, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047946963216424294, + "loss": 2.6104, + "theoretical_loss": 3.3081431636613363, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047904191616766467, + "loss": 2.6564, + "theoretical_loss": 3.3081326971384, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047861420017108645, + "loss": 2.682, + "theoretical_loss": 3.308122231172341, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004781864841745082, + "loss": 2.6996, + "theoretical_loss": 3.3081117657631056, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047775876817792985, + "loss": 2.7529, + "theoretical_loss": 3.3081013009106424, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004773310521813516, + "loss": 2.6481, + "theoretical_loss": 3.3080908366148973, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004769033361847733, + "loss": 2.5359, + "theoretical_loss": 3.308080372875819, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004764756201881951, + "loss": 2.6362, + "theoretical_loss": 3.3080699096933537, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047604790419161677, + "loss": 2.6062, + "theoretical_loss": 3.308059447067449, + "tokens_seen": 3154116608 + }, + { + "debugging/Self-BLEU-5": 0.4768917357816163, + "debugging/distinct-1-grams": 0.8069642973374905, + "debugging/distinct-2-grams": 0.9493042579915046, + "debugging/entropy-1-grams": 5.952991944891716, + "debugging/entropy-2-grams": 6.680805112573269, + "debugging/length": 514.4615384615385, + "debugging/num_segments": 13, + "debugging/score": 0.008350422819326409, + "debugging/score_std": 0.0069101491985588594, + "epoch": 0.53, + "objective/train/docs_used": 1728625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9037845134735107, + "objective/train/theoretical_loss": 3.3080489849980523, + "objective/train/tokens_used": 184298976, + "theoretical_loss": 3.3080489849980523, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004756201881950385, + "loss": 2.4958, + "theoretical_loss": 3.3080489849980523, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004751924721984602, + "loss": 2.7064, + "theoretical_loss": 3.3080385234851106, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047476475620188195, + "loss": 2.5776, + "theoretical_loss": 3.3080280625285714, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047433704020530374, + "loss": 2.7331, + "theoretical_loss": 3.308017602128382, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004739093242087254, + "loss": 2.6409, + "theoretical_loss": 3.308007142284489, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047348160821214714, + "loss": 2.664, + "theoretical_loss": 3.307996682996841, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047305389221556887, + "loss": 2.5432, + "theoretical_loss": 3.3079862242653846, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004726261762189906, + "loss": 2.5397, + "theoretical_loss": 3.307975766090067, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004721984602224123, + "loss": 2.4205, + "theoretical_loss": 3.3079653084708354, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047177074422583405, + "loss": 2.6373, + "theoretical_loss": 3.3079548514076373, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004713430282292558, + "loss": 2.763, + "theoretical_loss": 3.307944394900421, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.53, + "learning_rate": 0.0004709153122326775, + "loss": 2.4824, + "theoretical_loss": 3.307933938949132, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047048759623609924, + "loss": 2.7033, + "theoretical_loss": 3.307923483553719, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1729266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.431025743484497, + "objective/train/theoretical_loss": 3.3079182560644496, + "objective/train/tokens_used": 185937376, + "theoretical_loss": 3.3079182560644496, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00047005988023952097, + "loss": 2.5109, + "theoretical_loss": 3.307913028714129, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004696321642429427, + "loss": 2.7625, + "theoretical_loss": 3.3079025744303094, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004692044482463644, + "loss": 2.6764, + "theoretical_loss": 3.3078921207022076, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046877673224978616, + "loss": 2.5695, + "theoretical_loss": 3.307881667529771, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004683490162532079, + "loss": 2.471, + "theoretical_loss": 3.3078712149129466, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004679213002566296, + "loss": 2.7204, + "theoretical_loss": 3.3078607628516825, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046749358426005134, + "loss": 2.6621, + "theoretical_loss": 3.3078503113459257, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046706586826347307, + "loss": 2.4893, + "theoretical_loss": 3.307839860395623, + "tokens_seen": 3156869120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046663815226689474, + "loss": 2.5267, + "theoretical_loss": 3.307829410000723, + "tokens_seen": 3157000192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046621043627031653, + "loss": 2.5875, + "theoretical_loss": 3.307818960161173, + "tokens_seen": 3157131264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046578272027373826, + "loss": 2.6245, + "theoretical_loss": 3.307808510876919, + "tokens_seen": 3157262336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046535500427716, + "loss": 2.4861, + "theoretical_loss": 3.3077980621479104, + "tokens_seen": 3157393408 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1730350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526437997817993, + "objective/train/theoretical_loss": 3.307787613974093, + "objective/train/tokens_used": 187575776, + "theoretical_loss": 3.307787613974093, + "tokens_seen": 3157524480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004649272882805817, + "loss": 2.7171, + "theoretical_loss": 3.307787613974093, + "tokens_seen": 3157524480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004644995722840034, + "loss": 2.662, + "theoretical_loss": 3.3077771663554154, + "tokens_seen": 3157655552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046407185628742517, + "loss": 2.6495, + "theoretical_loss": 3.3077667192918243, + "tokens_seen": 3157786624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004636441402908469, + "loss": 2.2755, + "theoretical_loss": 3.307756272783268, + "tokens_seen": 3157917696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046321642429426863, + "loss": 2.6799, + "theoretical_loss": 3.3077458268296933, + "tokens_seen": 3158048768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004627887082976903, + "loss": 2.5681, + "theoretical_loss": 3.3077353814310477, + "tokens_seen": 3158179840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004623609923011121, + "loss": 2.6143, + "theoretical_loss": 3.3077249365872787, + "tokens_seen": 3158310912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0004619332763045338, + "loss": 2.4642, + "theoretical_loss": 3.307714492298334, + "tokens_seen": 3158441984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046150556030795554, + "loss": 2.6805, + "theoretical_loss": 3.307704048564161, + "tokens_seen": 3158573056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046107784431137727, + "loss": 2.5288, + "theoretical_loss": 3.3076936053847072, + "tokens_seen": 3158704128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046065012831479895, + "loss": 2.6243, + "theoretical_loss": 3.3076831627599206, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00046022241231822073, + "loss": 2.602, + "theoretical_loss": 3.307672720689748, + "tokens_seen": 3158966272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00045979469632164246, + "loss": 2.3506, + "theoretical_loss": 3.3076622791741377, + "tokens_seen": 3159097344 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1730945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.211646556854248, + "objective/train/theoretical_loss": 3.3076570586242764, + "objective/train/tokens_used": 189214176, + "theoretical_loss": 3.3076570586242764, + "tokens_seen": 3159162880 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004593669803250642, + "loss": 2.6011, + "theoretical_loss": 3.307651838213036, + "tokens_seen": 3159228416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045893926432848586, + "loss": 2.7573, + "theoretical_loss": 3.3076413978063917, + "tokens_seen": 3159359488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004585115483319076, + "loss": 2.7478, + "theoretical_loss": 3.307630957954152, + "tokens_seen": 3159490560 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004580838323353294, + "loss": 2.6339, + "theoretical_loss": 3.307620518656264, + "tokens_seen": 3159621632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004576561163387511, + "loss": 2.4148, + "theoretical_loss": 3.307610079912676, + "tokens_seen": 3159752704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045722840034217283, + "loss": 2.5499, + "theoretical_loss": 3.3075996417233346, + "tokens_seen": 3159883776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004568006843455945, + "loss": 2.5445, + "theoretical_loss": 3.3075892040881887, + "tokens_seen": 3160014848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045637296834901623, + "loss": 2.6046, + "theoretical_loss": 3.307578767007185, + "tokens_seen": 3160145920 + }, + { + "epoch": 0.55, + "learning_rate": 0.000455945252352438, + "loss": 2.5384, + "theoretical_loss": 3.307568330480271, + "tokens_seen": 3160276992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045551753635585975, + "loss": 2.6444, + "theoretical_loss": 3.3075578945073945, + "tokens_seen": 3160408064 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004550898203592814, + "loss": 2.5526, + "theoretical_loss": 3.3075474590885032, + "tokens_seen": 3160539136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045466210436270315, + "loss": 2.5814, + "theoretical_loss": 3.307537024223545, + "tokens_seen": 3160670208 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1732078, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.850566864013672, + "objective/train/theoretical_loss": 3.307526589912467, + "objective/train/tokens_used": 190852576, + "theoretical_loss": 3.307526589912467, + "tokens_seen": 3160801280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045423438836612493, + "loss": 2.6059, + "theoretical_loss": 3.307526589912467, + "tokens_seen": 3160801280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045380667236954666, + "loss": 2.5523, + "theoretical_loss": 3.3075161561552173, + "tokens_seen": 3160932352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045337895637296834, + "loss": 2.4169, + "theoretical_loss": 3.307505722951743, + "tokens_seen": 3161063424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045295124037639006, + "loss": 2.5638, + "theoretical_loss": 3.307495290301992, + "tokens_seen": 3161194496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004525235243798118, + "loss": 2.6393, + "theoretical_loss": 3.307484858205912, + "tokens_seen": 3161325568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004520958083832336, + "loss": 2.5884, + "theoretical_loss": 3.3074744266634513, + "tokens_seen": 3161456640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004516680923866553, + "loss": 2.4982, + "theoretical_loss": 3.3074639956745564, + "tokens_seen": 3161587712 + }, + { + "epoch": 0.55, + "learning_rate": 0.000451240376390077, + "loss": 2.6226, + "theoretical_loss": 3.3074535652391757, + "tokens_seen": 3161718784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004508126603934987, + "loss": 2.4801, + "theoretical_loss": 3.3074431353572566, + "tokens_seen": 3161849856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00045038494439692044, + "loss": 2.6883, + "theoretical_loss": 3.307432706028747, + "tokens_seen": 3161980928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0004499572284003422, + "loss": 2.6224, + "theoretical_loss": 3.307422277253594, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004495295124037639, + "loss": 2.5753, + "theoretical_loss": 3.3074118490317463, + "tokens_seen": 3162243072 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004491017964071856, + "loss": 2.5001, + "theoretical_loss": 3.307401421363151, + "tokens_seen": 3162374144 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1732556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.745497941970825, + "objective/train/theoretical_loss": 3.3073962077363066, + "objective/train/tokens_used": 192490976, + "theoretical_loss": 3.3073962077363066, + "tokens_seen": 3162439680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044867408041060735, + "loss": 2.5138, + "theoretical_loss": 3.307390994247756, + "tokens_seen": 3162505216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004482463644140291, + "loss": 2.5034, + "theoretical_loss": 3.3073805676855086, + "tokens_seen": 3162636288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044781864841745086, + "loss": 2.6515, + "theoretical_loss": 3.3073701416763575, + "tokens_seen": 3162767360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044739093242087254, + "loss": 2.6301, + "theoretical_loss": 3.3073597162202493, + "tokens_seen": 3162898432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044696321642429427, + "loss": 2.5188, + "theoretical_loss": 3.3073492913171325, + "tokens_seen": 3163029504 + }, + { + "epoch": 0.56, + "learning_rate": 0.000446535500427716, + "loss": 2.6919, + "theoretical_loss": 3.307338866966955, + "tokens_seen": 3163160576 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004461077844311378, + "loss": 2.7455, + "theoretical_loss": 3.3073284431696632, + "tokens_seen": 3163291648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044568006843455945, + "loss": 2.6212, + "theoretical_loss": 3.3073180199252064, + "tokens_seen": 3163422720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004452523524379812, + "loss": 2.6162, + "theoretical_loss": 3.3073075972335317, + "tokens_seen": 3163553792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004448246364414029, + "loss": 2.6251, + "theoretical_loss": 3.307297175094587, + "tokens_seen": 3163684864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044439692044482464, + "loss": 2.6537, + "theoretical_loss": 3.3072867535083206, + "tokens_seen": 3163815936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004439692044482464, + "loss": 2.5382, + "theoretical_loss": 3.3072763324746792, + "tokens_seen": 3163947008 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1733728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.713740825653076, + "objective/train/theoretical_loss": 3.3072659119936114, + "objective/train/tokens_used": 194129376, + "theoretical_loss": 3.3072659119936114, + "tokens_seen": 3164078080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004435414884516681, + "loss": 2.4157, + "theoretical_loss": 3.3072659119936114, + "tokens_seen": 3164078080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004431137724550898, + "loss": 2.4885, + "theoretical_loss": 3.3072554920650648, + "tokens_seen": 3164209152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044268605645851155, + "loss": 2.6095, + "theoretical_loss": 3.3072450726889873, + "tokens_seen": 3164340224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004422583404619333, + "loss": 2.6318, + "theoretical_loss": 3.3072346538653266, + "tokens_seen": 3164471296 + }, + { + "epoch": 0.56, + "learning_rate": 0.000441830624465355, + "loss": 2.473, + "theoretical_loss": 3.307224235594031, + "tokens_seen": 3164602368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044140290846877674, + "loss": 2.5879, + "theoretical_loss": 3.3072138178750476, + "tokens_seen": 3164733440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044097519247219847, + "loss": 2.4492, + "theoretical_loss": 3.3072034007083246, + "tokens_seen": 3164864512 + }, + { + "epoch": 0.56, + "learning_rate": 0.0004405474764756202, + "loss": 2.6278, + "theoretical_loss": 3.3071929840938097, + "tokens_seen": 3164995584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00044011976047904193, + "loss": 2.6007, + "theoretical_loss": 3.307182568031451, + "tokens_seen": 3165126656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00043969204448246366, + "loss": 2.4395, + "theoretical_loss": 3.307172152521196, + "tokens_seen": 3165257728 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004392643284858854, + "loss": 2.4231, + "theoretical_loss": 3.307161737562993, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004388366124893071, + "loss": 2.4642, + "theoretical_loss": 3.30715132315679, + "tokens_seen": 3165519872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043840889649272884, + "loss": 2.5576, + "theoretical_loss": 3.307140909302534, + "tokens_seen": 3165650944 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1734416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.445899248123169, + "objective/train/theoretical_loss": 3.3071357025823707, + "objective/train/tokens_used": 195767776, + "theoretical_loss": 3.3071357025823707, + "tokens_seen": 3165716480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043798118049615057, + "loss": 2.6092, + "theoretical_loss": 3.307130496000174, + "tokens_seen": 3165782016 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004375534644995723, + "loss": 2.522, + "theoretical_loss": 3.3071200832496577, + "tokens_seen": 3165913088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043712574850299403, + "loss": 2.6278, + "theoretical_loss": 3.3071096710509322, + "tokens_seen": 3166044160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043669803250641576, + "loss": 2.531, + "theoretical_loss": 3.3070992594039463, + "tokens_seen": 3166175232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043627031650983743, + "loss": 2.5041, + "theoretical_loss": 3.307088848308647, + "tokens_seen": 3166306304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004358426005132592, + "loss": 2.5839, + "theoretical_loss": 3.307078437764983, + "tokens_seen": 3166437376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043541488451668094, + "loss": 2.5721, + "theoretical_loss": 3.3070680277729023, + "tokens_seen": 3166568448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043498716852010267, + "loss": 2.4703, + "theoretical_loss": 3.307057618332353, + "tokens_seen": 3166699520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004345594525235244, + "loss": 2.5753, + "theoretical_loss": 3.3070472094432817, + "tokens_seen": 3166830592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004341317365269461, + "loss": 2.5399, + "theoretical_loss": 3.307036801105638, + "tokens_seen": 3166961664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043370402053036786, + "loss": 2.4816, + "theoretical_loss": 3.3070263933193687, + "tokens_seen": 3167092736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004332763045337896, + "loss": 2.7456, + "theoretical_loss": 3.307015986084422, + "tokens_seen": 3167223808 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1735648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1868679523468018, + "objective/train/theoretical_loss": 3.3070055794007467, + "objective/train/tokens_used": 197406176, + "theoretical_loss": 3.3070055794007467, + "tokens_seen": 3167354880 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004328485885372113, + "loss": 2.4637, + "theoretical_loss": 3.3070055794007467, + "tokens_seen": 3167354880 + }, + { + "epoch": 0.57, + "learning_rate": 0.000432420872540633, + "loss": 2.638, + "theoretical_loss": 3.3069951732682896, + "tokens_seen": 3167485952 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004319931565440547, + "loss": 2.603, + "theoretical_loss": 3.3069847676869997, + "tokens_seen": 3167617024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004315654405474765, + "loss": 2.6497, + "theoretical_loss": 3.306974362656825, + "tokens_seen": 3167748096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043113772455089823, + "loss": 2.5017, + "theoretical_loss": 3.3069639581777124, + "tokens_seen": 3167879168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043071000855431996, + "loss": 2.6438, + "theoretical_loss": 3.3069535542496107, + "tokens_seen": 3168010240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00043028229255774163, + "loss": 2.7575, + "theoretical_loss": 3.3069431508724683, + "tokens_seen": 3168141312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0004298545765611634, + "loss": 2.4662, + "theoretical_loss": 3.3069327480462323, + "tokens_seen": 3168272384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00042942686056458515, + "loss": 2.5272, + "theoretical_loss": 3.3069223457708516, + "tokens_seen": 3168403456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004289991445680069, + "loss": 2.5104, + "theoretical_loss": 3.3069119440462735, + "tokens_seen": 3168534528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042857142857142855, + "loss": 2.5812, + "theoretical_loss": 3.306901542872447, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004281437125748503, + "loss": 2.4954, + "theoretical_loss": 3.306891142249319, + "tokens_seen": 3168796672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042771599657827206, + "loss": 2.5327, + "theoretical_loss": 3.3068807421768383, + "tokens_seen": 3168927744 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1736895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.836125135421753, + "objective/train/theoretical_loss": 3.306875542347074, + "objective/train/tokens_used": 199044576, + "theoretical_loss": 3.306875542347074, + "tokens_seen": 3168993280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004272882805816938, + "loss": 2.6185, + "theoretical_loss": 3.306870342654953, + "tokens_seen": 3169058816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004268605645851155, + "loss": 2.6293, + "theoretical_loss": 3.3068599436836106, + "tokens_seen": 3169189888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004264328485885372, + "loss": 2.5881, + "theoretical_loss": 3.30684954526276, + "tokens_seen": 3169320960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004260051325919589, + "loss": 2.5863, + "theoretical_loss": 3.306839147392348, + "tokens_seen": 3169452032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004255774165953807, + "loss": 2.5838, + "theoretical_loss": 3.3068287500723246, + "tokens_seen": 3169583104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042514970059880243, + "loss": 2.6051, + "theoretical_loss": 3.3068183533026367, + "tokens_seen": 3169714176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004247219846022241, + "loss": 2.5963, + "theoretical_loss": 3.3068079570832323, + "tokens_seen": 3169845248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042429426860564584, + "loss": 2.5536, + "theoretical_loss": 3.30679756141406, + "tokens_seen": 3169976320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042386655260906757, + "loss": 2.4996, + "theoretical_loss": 3.3067871662950674, + "tokens_seen": 3170107392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042343883661248935, + "loss": 2.5818, + "theoretical_loss": 3.3067767717262035, + "tokens_seen": 3170238464 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004230111206159111, + "loss": 2.6423, + "theoretical_loss": 3.3067663777074157, + "tokens_seen": 3170369536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042258340461933275, + "loss": 2.6811, + "theoretical_loss": 3.3067559842386522, + "tokens_seen": 3170500608 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1737490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8579704761505127, + "objective/train/theoretical_loss": 3.306745591319862, + "objective/train/tokens_used": 200682976, + "theoretical_loss": 3.306745591319862, + "tokens_seen": 3170631680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004221556886227545, + "loss": 2.5963, + "theoretical_loss": 3.306745591319862, + "tokens_seen": 3170631680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042172797262617626, + "loss": 2.6279, + "theoretical_loss": 3.3067351989509914, + "tokens_seen": 3170762752 + }, + { + "epoch": 0.58, + "learning_rate": 0.000421300256629598, + "loss": 2.5459, + "theoretical_loss": 3.3067248071319906, + "tokens_seen": 3170893824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00042087254063301967, + "loss": 2.5876, + "theoretical_loss": 3.306714415862807, + "tokens_seen": 3171024896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004204448246364414, + "loss": 2.6607, + "theoretical_loss": 3.3067040251433886, + "tokens_seen": 3171155968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004200171086398631, + "loss": 2.5979, + "theoretical_loss": 3.3066936349736835, + "tokens_seen": 3171287040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0004195893926432849, + "loss": 2.4231, + "theoretical_loss": 3.3066832453536406, + "tokens_seen": 3171418112 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004191616766467066, + "loss": 2.4679, + "theoretical_loss": 3.306672856283207, + "tokens_seen": 3171549184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004187339606501283, + "loss": 2.6636, + "theoretical_loss": 3.306662467762332, + "tokens_seen": 3171680256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041830624465355004, + "loss": 2.413, + "theoretical_loss": 3.3066520797909633, + "tokens_seen": 3171811328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041787852865697177, + "loss": 2.5809, + "theoretical_loss": 3.306641692369049, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041745081266039355, + "loss": 2.5255, + "theoretical_loss": 3.3066313054965377, + "tokens_seen": 3172073472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004170230966638152, + "loss": 2.6016, + "theoretical_loss": 3.3066209191733775, + "tokens_seen": 3172204544 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1738634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.700066566467285, + "objective/train/theoretical_loss": 3.306615726217788, + "objective/train/tokens_used": 202321376, + "theoretical_loss": 3.306615726217788, + "tokens_seen": 3172270080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041659538066723695, + "loss": 2.5632, + "theoretical_loss": 3.3066105333995166, + "tokens_seen": 3172335616 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004161676646706587, + "loss": 2.5207, + "theoretical_loss": 3.306600148174903, + "tokens_seen": 3172466688 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004157399486740804, + "loss": 2.5567, + "theoretical_loss": 3.3065897634994856, + "tokens_seen": 3172597760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041531223267750214, + "loss": 2.4908, + "theoretical_loss": 3.306579379373212, + "tokens_seen": 3172728832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041488451668092387, + "loss": 2.6118, + "theoretical_loss": 3.306568995796031, + "tokens_seen": 3172859904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004144568006843456, + "loss": 2.6527, + "theoretical_loss": 3.3065586127678905, + "tokens_seen": 3172990976 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004140290846877673, + "loss": 2.5829, + "theoretical_loss": 3.3065482302887395, + "tokens_seen": 3173122048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004136013686911891, + "loss": 2.5831, + "theoretical_loss": 3.3065378483585253, + "tokens_seen": 3173253120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004131736526946108, + "loss": 2.6005, + "theoretical_loss": 3.3065274669771965, + "tokens_seen": 3173384192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004127459366980325, + "loss": 2.5328, + "theoretical_loss": 3.3065170861447015, + "tokens_seen": 3173515264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041231822070145424, + "loss": 2.5861, + "theoretical_loss": 3.306506705860989, + "tokens_seen": 3173646336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041189050470487597, + "loss": 2.5119, + "theoretical_loss": 3.306496326126007, + "tokens_seen": 3173777408 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1739179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4293901920318604, + "objective/train/theoretical_loss": 3.306485946939704, + "objective/train/tokens_used": 203959776, + "theoretical_loss": 3.306485946939704, + "tokens_seen": 3173908480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004114627887082977, + "loss": 2.5072, + "theoretical_loss": 3.306485946939704, + "tokens_seen": 3173908480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041103507271171943, + "loss": 2.5568, + "theoretical_loss": 3.306475568302028, + "tokens_seen": 3174039552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00041060735671514116, + "loss": 2.6288, + "theoretical_loss": 3.3064651902129274, + "tokens_seen": 3174170624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004101796407185629, + "loss": 2.7253, + "theoretical_loss": 3.3064548126723508, + "tokens_seen": 3174301696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0004097519247219846, + "loss": 2.6307, + "theoretical_loss": 3.3064444356802465, + "tokens_seen": 3174432768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00040932420872540634, + "loss": 2.5171, + "theoretical_loss": 3.3064340592365626, + "tokens_seen": 3174563840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040889649272882807, + "loss": 2.4488, + "theoretical_loss": 3.306423683341248, + "tokens_seen": 3174694912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004084687767322498, + "loss": 2.4653, + "theoretical_loss": 3.3064133079942506, + "tokens_seen": 3174825984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040804106073567153, + "loss": 2.4555, + "theoretical_loss": 3.3064029331955194, + "tokens_seen": 3174957056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004076133447390932, + "loss": 2.5712, + "theoretical_loss": 3.3063925589450016, + "tokens_seen": 3175088128 + }, + { + "epoch": 0.6, + "learning_rate": 0.000407185628742515, + "loss": 2.5779, + "theoretical_loss": 3.3063821852426467, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004067579127459367, + "loss": 2.5305, + "theoretical_loss": 3.306371812088403, + "tokens_seen": 3175350272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040633019674935844, + "loss": 2.5672, + "theoretical_loss": 3.3063614394822185, + "tokens_seen": 3175481344 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1739867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5515024662017822, + "objective/train/theoretical_loss": 3.3063562533846325, + "objective/train/tokens_used": 205598176, + "theoretical_loss": 3.3063562533846325, + "tokens_seen": 3175546880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004059024807527802, + "loss": 2.6056, + "theoretical_loss": 3.3063510674240417, + "tokens_seen": 3175612416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040547476475620185, + "loss": 2.5911, + "theoretical_loss": 3.3063406959138213, + "tokens_seen": 3175743488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040504704875962363, + "loss": 2.4831, + "theoretical_loss": 3.3063303249515057, + "tokens_seen": 3175874560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040461933276304536, + "loss": 2.7188, + "theoretical_loss": 3.3063199545370425, + "tokens_seen": 3176005632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004041916167664671, + "loss": 2.5182, + "theoretical_loss": 3.3063095846703816, + "tokens_seen": 3176136704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040376390076988876, + "loss": 2.5775, + "theoretical_loss": 3.3062992153514705, + "tokens_seen": 3176267776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040333618477331055, + "loss": 2.5908, + "theoretical_loss": 3.306288846580258, + "tokens_seen": 3176398848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004029084687767323, + "loss": 2.5978, + "theoretical_loss": 3.3062784783566923, + "tokens_seen": 3176529920 + }, + { + "epoch": 0.6, + "learning_rate": 0.000402480752780154, + "loss": 2.5184, + "theoretical_loss": 3.306268110680722, + "tokens_seen": 3176660992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004020530367835757, + "loss": 2.5438, + "theoretical_loss": 3.306257743552296, + "tokens_seen": 3176792064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004016253207869974, + "loss": 2.4912, + "theoretical_loss": 3.306247376971362, + "tokens_seen": 3176923136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004011976047904192, + "loss": 2.6222, + "theoretical_loss": 3.3062370109378696, + "tokens_seen": 3177054208 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1740434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8082294464111328, + "objective/train/theoretical_loss": 3.306226645451766, + "objective/train/tokens_used": 207236576, + "theoretical_loss": 3.306226645451766, + "tokens_seen": 3177185280 + }, + { + "epoch": 0.6, + "learning_rate": 0.0004007698887938409, + "loss": 2.3938, + "theoretical_loss": 3.306226645451766, + "tokens_seen": 3177185280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00040034217279726265, + "loss": 2.5175, + "theoretical_loss": 3.3062162805130004, + "tokens_seen": 3177316352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003999144568006843, + "loss": 2.4586, + "theoretical_loss": 3.3062059161215216, + "tokens_seen": 3177447424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00039948674080410605, + "loss": 2.6279, + "theoretical_loss": 3.3061955522772775, + "tokens_seen": 3177578496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00039905902480752783, + "loss": 2.5878, + "theoretical_loss": 3.306185188980217, + "tokens_seen": 3177709568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039863130881094956, + "loss": 2.3969, + "theoretical_loss": 3.3061748262302886, + "tokens_seen": 3177840640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039820359281437124, + "loss": 2.6624, + "theoretical_loss": 3.306164464027441, + "tokens_seen": 3177971712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039777587681779297, + "loss": 2.6423, + "theoretical_loss": 3.3061541023716225, + "tokens_seen": 3178102784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039734816082121475, + "loss": 2.3862, + "theoretical_loss": 3.3061437412627814, + "tokens_seen": 3178233856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003969204448246365, + "loss": 2.5846, + "theoretical_loss": 3.306133380700867, + "tokens_seen": 3178364928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003964927288280582, + "loss": 2.7634, + "theoretical_loss": 3.306123020685827, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003960650128314799, + "loss": 2.6097, + "theoretical_loss": 3.306112661217611, + "tokens_seen": 3178627072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003956372968349016, + "loss": 2.5263, + "theoretical_loss": 3.306102302296167, + "tokens_seen": 3178758144 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1741553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5916895866394043, + "objective/train/theoretical_loss": 3.3060971230404683, + "objective/train/tokens_used": 208874976, + "theoretical_loss": 3.3060971230404683, + "tokens_seen": 3178823680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003952095808383234, + "loss": 2.5908, + "theoretical_loss": 3.3060919439214436, + "tokens_seen": 3178889216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003947818648417451, + "loss": 2.492, + "theoretical_loss": 3.3060815860933896, + "tokens_seen": 3179020288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003943541488451668, + "loss": 2.5445, + "theoretical_loss": 3.306071228811953, + "tokens_seen": 3179151360 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003939264328485885, + "loss": 2.569, + "theoretical_loss": 3.3060608720770834, + "tokens_seen": 3179282432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039349871685201025, + "loss": 2.537, + "theoretical_loss": 3.306050515888729, + "tokens_seen": 3179413504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039307100085543204, + "loss": 2.5606, + "theoretical_loss": 3.306040160246838, + "tokens_seen": 3179544576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039264328485885376, + "loss": 2.6501, + "theoretical_loss": 3.3060298051513595, + "tokens_seen": 3179675648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039221556886227544, + "loss": 2.7225, + "theoretical_loss": 3.306019450602242, + "tokens_seen": 3179806720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039178785286569717, + "loss": 2.5038, + "theoretical_loss": 3.306009096599434, + "tokens_seen": 3179937792 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003913601368691189, + "loss": 2.6857, + "theoretical_loss": 3.3059987431428848, + "tokens_seen": 3180068864 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003909324208725407, + "loss": 2.5266, + "theoretical_loss": 3.3059883902325424, + "tokens_seen": 3180199936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00039050470487596235, + "loss": 2.5905, + "theoretical_loss": 3.3059780378683556, + "tokens_seen": 3180331008 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1742088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.464376449584961, + "objective/train/theoretical_loss": 3.3059676860502734, + "objective/train/tokens_used": 210513376, + "theoretical_loss": 3.3059676860502734, + "tokens_seen": 3180462080 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003900769888793841, + "loss": 2.5927, + "theoretical_loss": 3.3059676860502734, + "tokens_seen": 3180462080 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003896492728828058, + "loss": 2.6167, + "theoretical_loss": 3.305957334778244, + "tokens_seen": 3180593152 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003892215568862276, + "loss": 2.5417, + "theoretical_loss": 3.3059469840522167, + "tokens_seen": 3180724224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003887938408896493, + "loss": 2.589, + "theoretical_loss": 3.3059366338721397, + "tokens_seen": 3180855296 + }, + { + "epoch": 0.62, + "learning_rate": 0.000388366124893071, + "loss": 2.502, + "theoretical_loss": 3.305926284237962, + "tokens_seen": 3180986368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003879384088964927, + "loss": 2.5827, + "theoretical_loss": 3.305915935149632, + "tokens_seen": 3181117440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038751069289991446, + "loss": 2.5516, + "theoretical_loss": 3.3059055866070985, + "tokens_seen": 3181248512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038708297690333624, + "loss": 2.4978, + "theoretical_loss": 3.3058952386103106, + "tokens_seen": 3181379584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003866552609067579, + "loss": 2.505, + "theoretical_loss": 3.305884891159217, + "tokens_seen": 3181510656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038622754491017964, + "loss": 2.4143, + "theoretical_loss": 3.305874544253766, + "tokens_seen": 3181641728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038579982891360137, + "loss": 2.643, + "theoretical_loss": 3.3058641978939063, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003853721129170231, + "loss": 2.6154, + "theoretical_loss": 3.305853852079587, + "tokens_seen": 3181903872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038494439692044483, + "loss": 2.7237, + "theoretical_loss": 3.305843506810757, + "tokens_seen": 3182034944 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1743107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.488473415374756, + "objective/train/theoretical_loss": 3.3058383343808844, + "objective/train/tokens_used": 212151776, + "theoretical_loss": 3.3058383343808844, + "tokens_seen": 3182100480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038451668092386656, + "loss": 2.6367, + "theoretical_loss": 3.3058331620873647, + "tokens_seen": 3182166016 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003840889649272883, + "loss": 2.6144, + "theoretical_loss": 3.305822817909359, + "tokens_seen": 3182297088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038366124893071, + "loss": 2.4897, + "theoretical_loss": 3.305812474276689, + "tokens_seen": 3182428160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038323353293413174, + "loss": 2.6743, + "theoretical_loss": 3.305802131189303, + "tokens_seen": 3182559232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038280581693755347, + "loss": 2.5733, + "theoretical_loss": 3.30579178864715, + "tokens_seen": 3182690304 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003823781009409752, + "loss": 2.6581, + "theoretical_loss": 3.3057814466501787, + "tokens_seen": 3182821376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038195038494439693, + "loss": 2.6084, + "theoretical_loss": 3.3057711051983385, + "tokens_seen": 3182952448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038152266894781866, + "loss": 2.6092, + "theoretical_loss": 3.3057607642915774, + "tokens_seen": 3183083520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038109495295124033, + "loss": 2.4866, + "theoretical_loss": 3.3057504239298448, + "tokens_seen": 3183214592 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003806672369546621, + "loss": 2.4784, + "theoretical_loss": 3.305740084113089, + "tokens_seen": 3183345664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00038023952095808384, + "loss": 2.4723, + "theoretical_loss": 3.3057297448412593, + "tokens_seen": 3183476736 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003798118049615056, + "loss": 2.8141, + "theoretical_loss": 3.3057194061143047, + "tokens_seen": 3183607808 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1744349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5482563972473145, + "objective/train/theoretical_loss": 3.305709067932173, + "objective/train/tokens_used": 213790176, + "theoretical_loss": 3.305709067932173, + "tokens_seen": 3183738880 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003793840889649273, + "loss": 2.5627, + "theoretical_loss": 3.305709067932173, + "tokens_seen": 3183738880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00037895637296834903, + "loss": 2.5551, + "theoretical_loss": 3.3056987302948144, + "tokens_seen": 3183869952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037852865697177076, + "loss": 2.6401, + "theoretical_loss": 3.3056883932021774, + "tokens_seen": 3184001024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003781009409751925, + "loss": 2.5753, + "theoretical_loss": 3.30567805665421, + "tokens_seen": 3184132096 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003776732249786142, + "loss": 2.6404, + "theoretical_loss": 3.3056677206508622, + "tokens_seen": 3184263168 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003772455089820359, + "loss": 2.6395, + "theoretical_loss": 3.3056573851920827, + "tokens_seen": 3184394240 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003768177929854577, + "loss": 2.7381, + "theoretical_loss": 3.3056470502778197, + "tokens_seen": 3184525312 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003763900769888794, + "loss": 2.4904, + "theoretical_loss": 3.3056367159080224, + "tokens_seen": 3184656384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037596236099230113, + "loss": 2.6728, + "theoretical_loss": 3.3056263820826404, + "tokens_seen": 3184787456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037553464499572286, + "loss": 2.5823, + "theoretical_loss": 3.3056160488016215, + "tokens_seen": 3184918528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037510692899914453, + "loss": 2.5669, + "theoretical_loss": 3.3056057160649153, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003746792130025663, + "loss": 2.4694, + "theoretical_loss": 3.3055953838724705, + "tokens_seen": 3185180672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037425149700598805, + "loss": 2.5418, + "theoretical_loss": 3.3055850522242363, + "tokens_seen": 3185311744 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1745003, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1426408290863037, + "objective/train/theoretical_loss": 3.3055798866041823, + "objective/train/tokens_used": 215428576, + "theoretical_loss": 3.3055798866041823, + "tokens_seen": 3185377280 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003738237810094098, + "loss": 2.471, + "theoretical_loss": 3.305574721120162, + "tokens_seen": 3185442816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037339606501283145, + "loss": 2.5397, + "theoretical_loss": 3.3055643905601952, + "tokens_seen": 3185573888 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003729683490162532, + "loss": 2.5313, + "theoretical_loss": 3.305554060544286, + "tokens_seen": 3185704960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00037254063301967496, + "loss": 2.4349, + "theoretical_loss": 3.305543731072383, + "tokens_seen": 3185836032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003721129170230967, + "loss": 2.7246, + "theoretical_loss": 3.3055334021444356, + "tokens_seen": 3185967104 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003716852010265184, + "loss": 2.5888, + "theoretical_loss": 3.305523073760392, + "tokens_seen": 3186098176 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003712574850299401, + "loss": 2.5187, + "theoretical_loss": 3.305512745920202, + "tokens_seen": 3186229248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003708297690333619, + "loss": 2.555, + "theoretical_loss": 3.305502418623814, + "tokens_seen": 3186360320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003704020530367836, + "loss": 2.585, + "theoretical_loss": 3.305492091871177, + "tokens_seen": 3186491392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00036997433704020533, + "loss": 2.5148, + "theoretical_loss": 3.3054817656622406, + "tokens_seen": 3186622464 + }, + { + "epoch": 0.63, + "learning_rate": 0.000369546621043627, + "loss": 2.6244, + "theoretical_loss": 3.305471439996953, + "tokens_seen": 3186753536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00036911890504704874, + "loss": 2.4488, + "theoretical_loss": 3.3054611148752637, + "tokens_seen": 3186884608 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1746088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6003615856170654, + "objective/train/theoretical_loss": 3.3054507902971215, + "objective/train/tokens_used": 217066976, + "theoretical_loss": 3.3054507902971215, + "tokens_seen": 3187015680 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003686911890504705, + "loss": 2.6781, + "theoretical_loss": 3.3054507902971215, + "tokens_seen": 3187015680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036826347305389225, + "loss": 2.5646, + "theoretical_loss": 3.305440466262476, + "tokens_seen": 3187146752 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003678357570573139, + "loss": 2.5774, + "theoretical_loss": 3.3054301427712756, + "tokens_seen": 3187277824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036740804106073565, + "loss": 2.5979, + "theoretical_loss": 3.3054198198234697, + "tokens_seen": 3187408896 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003669803250641574, + "loss": 2.5136, + "theoretical_loss": 3.305409497419007, + "tokens_seen": 3187539968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036655260906757916, + "loss": 2.4793, + "theoretical_loss": 3.305399175557837, + "tokens_seen": 3187671040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003661248930710009, + "loss": 2.7727, + "theoretical_loss": 3.3053888542399084, + "tokens_seen": 3187802112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036569717707442257, + "loss": 2.5126, + "theoretical_loss": 3.3053785334651704, + "tokens_seen": 3187933184 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003652694610778443, + "loss": 2.6187, + "theoretical_loss": 3.305368213233572, + "tokens_seen": 3188064256 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003648417450812661, + "loss": 2.4938, + "theoretical_loss": 3.3053578935450627, + "tokens_seen": 3188195328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003644140290846878, + "loss": 2.5505, + "theoretical_loss": 3.3053475743995913, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003639863130881095, + "loss": 2.5736, + "theoretical_loss": 3.305337255797107, + "tokens_seen": 3188457472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003635585970915312, + "loss": 2.6112, + "theoretical_loss": 3.3053269377375587, + "tokens_seen": 3188588544 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1746515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.432135581970215, + "objective/train/theoretical_loss": 3.3053217789113694, + "objective/train/tokens_used": 218705376, + "theoretical_loss": 3.3053217789113694, + "tokens_seen": 3188654080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036313088109495294, + "loss": 2.6224, + "theoretical_loss": 3.3053166202208955, + "tokens_seen": 3188719616 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003627031650983747, + "loss": 2.7255, + "theoretical_loss": 3.3053063032470664, + "tokens_seen": 3188850688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036227544910179645, + "loss": 2.8025, + "theoretical_loss": 3.305295986816021, + "tokens_seen": 3188981760 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003618477331052181, + "loss": 2.5396, + "theoretical_loss": 3.3052856709277085, + "tokens_seen": 3189112832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036142001710863986, + "loss": 2.5454, + "theoretical_loss": 3.3052753555820775, + "tokens_seen": 3189243904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003609923011120616, + "loss": 2.578, + "theoretical_loss": 3.3052650407790773, + "tokens_seen": 3189374976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036056458511548337, + "loss": 2.4681, + "theoretical_loss": 3.3052547265186574, + "tokens_seen": 3189506048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00036013686911890504, + "loss": 2.5775, + "theoretical_loss": 3.3052444128007665, + "tokens_seen": 3189637120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00035970915312232677, + "loss": 2.5996, + "theoretical_loss": 3.305234099625354, + "tokens_seen": 3189768192 + }, + { + "epoch": 0.64, + "learning_rate": 0.0003592814371257485, + "loss": 2.5099, + "theoretical_loss": 3.3052237869923693, + "tokens_seen": 3189899264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00035885372112917023, + "loss": 2.6524, + "theoretical_loss": 3.305213474901761, + "tokens_seen": 3190030336 + }, + { + "epoch": 0.65, + "learning_rate": 0.000358426005132592, + "loss": 2.7845, + "theoretical_loss": 3.305203163353479, + "tokens_seen": 3190161408 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1747620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.397430419921875, + "objective/train/theoretical_loss": 3.305192852347472, + "objective/train/tokens_used": 220343776, + "theoretical_loss": 3.305192852347472, + "tokens_seen": 3190292480 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003579982891360137, + "loss": 2.6162, + "theoretical_loss": 3.305192852347472, + "tokens_seen": 3190292480 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003575705731394354, + "loss": 2.5697, + "theoretical_loss": 3.3051825418836893, + "tokens_seen": 3190423552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035714285714285714, + "loss": 2.441, + "theoretical_loss": 3.3051722319620804, + "tokens_seen": 3190554624 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003567151411462789, + "loss": 2.6218, + "theoretical_loss": 3.305161922582594, + "tokens_seen": 3190685696 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003562874251497006, + "loss": 2.6723, + "theoretical_loss": 3.3051516137451795, + "tokens_seen": 3190816768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035585970915312233, + "loss": 2.634, + "theoretical_loss": 3.3051413054497867, + "tokens_seen": 3190947840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035543199315654406, + "loss": 2.6185, + "theoretical_loss": 3.305130997696364, + "tokens_seen": 3191078912 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003550042771599658, + "loss": 2.6652, + "theoretical_loss": 3.3051206904848613, + "tokens_seen": 3191209984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035457656116338757, + "loss": 2.6291, + "theoretical_loss": 3.3051103838152276, + "tokens_seen": 3191341056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035414884516680924, + "loss": 2.5576, + "theoretical_loss": 3.3051000776874115, + "tokens_seen": 3191472128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035372112917023097, + "loss": 2.6551, + "theoretical_loss": 3.3050897721013635, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003532934131736527, + "loss": 2.6107, + "theoretical_loss": 3.3050794670570323, + "tokens_seen": 3191734272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035286569717707443, + "loss": 2.6733, + "theoretical_loss": 3.305069162554367, + "tokens_seen": 3191865344 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1748281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.162297010421753, + "objective/train/theoretical_loss": 3.3050640105061433, + "objective/train/tokens_used": 221982176, + "theoretical_loss": 3.3050640105061433, + "tokens_seen": 3191930880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035243798118049616, + "loss": 2.8178, + "theoretical_loss": 3.3050588585933167, + "tokens_seen": 3191996416 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003520102651839179, + "loss": 2.5666, + "theoretical_loss": 3.3050485551738316, + "tokens_seen": 3192127488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003515825491873396, + "loss": 2.6362, + "theoretical_loss": 3.30503825229586, + "tokens_seen": 3192258560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00035115483319076135, + "loss": 2.4342, + "theoretical_loss": 3.305027949959352, + "tokens_seen": 3192389632 + }, + { + "epoch": 0.65, + "learning_rate": 0.000350727117194183, + "loss": 2.6797, + "theoretical_loss": 3.3050176481642564, + "tokens_seen": 3192520704 + }, + { + "epoch": 0.65, + "learning_rate": 0.0003502994011976048, + "loss": 2.558, + "theoretical_loss": 3.3050073469105223, + "tokens_seen": 3192651776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00034987168520102653, + "loss": 2.4285, + "theoretical_loss": 3.3049970461980998, + "tokens_seen": 3192782848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00034944396920444826, + "loss": 2.5022, + "theoretical_loss": 3.304986746026938, + "tokens_seen": 3192913920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00034901625320787, + "loss": 2.5511, + "theoretical_loss": 3.304976446396986, + "tokens_seen": 3193044992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00034858853721129166, + "loss": 2.5356, + "theoretical_loss": 3.304966147308193, + "tokens_seen": 3193176064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034816082121471345, + "loss": 2.5905, + "theoretical_loss": 3.3049558487605086, + "tokens_seen": 3193307136 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003477331052181352, + "loss": 2.6129, + "theoretical_loss": 3.3049455507538825, + "tokens_seen": 3193438208 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1749492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6648504734039307, + "objective/train/theoretical_loss": 3.304935253288263, + "objective/train/tokens_used": 223620576, + "theoretical_loss": 3.304935253288263, + "tokens_seen": 3193569280 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003473053892215569, + "loss": 2.6253, + "theoretical_loss": 3.304935253288263, + "tokens_seen": 3193569280 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003468776732249786, + "loss": 2.5282, + "theoretical_loss": 3.304924956363601, + "tokens_seen": 3193700352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034644995722840036, + "loss": 2.6169, + "theoretical_loss": 3.3049146599798447, + "tokens_seen": 3193831424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003460222412318221, + "loss": 2.471, + "theoretical_loss": 3.3049043641369438, + "tokens_seen": 3193962496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003455945252352438, + "loss": 2.5422, + "theoretical_loss": 3.304894068834848, + "tokens_seen": 3194093568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034516680923866555, + "loss": 2.4849, + "theoretical_loss": 3.3048837740735064, + "tokens_seen": 3194224640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003447390932420872, + "loss": 2.5686, + "theoretical_loss": 3.3048734798528683, + "tokens_seen": 3194355712 + }, + { + "epoch": 0.66, + "learning_rate": 0.000344311377245509, + "loss": 2.5296, + "theoretical_loss": 3.3048631861728834, + "tokens_seen": 3194486784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034388366124893073, + "loss": 2.4458, + "theoretical_loss": 3.304852893033501, + "tokens_seen": 3194617856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034345594525235246, + "loss": 2.674, + "theoretical_loss": 3.3048426004346707, + "tokens_seen": 3194748928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034302822925577414, + "loss": 2.66, + "theoretical_loss": 3.3048323083763416, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034260051325919587, + "loss": 2.444, + "theoretical_loss": 3.304822016858463, + "tokens_seen": 3195011072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00034217279726261765, + "loss": 2.492, + "theoretical_loss": 3.304811725880985, + "tokens_seen": 3195142144 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1749977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.884591817855835, + "objective/train/theoretical_loss": 3.3048065805948803, + "objective/train/tokens_used": 225258976, + "theoretical_loss": 3.3048065805948803, + "tokens_seen": 3195207680 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003417450812660394, + "loss": 2.6708, + "theoretical_loss": 3.304801435443857, + "tokens_seen": 3195273216 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003413173652694611, + "loss": 2.6639, + "theoretical_loss": 3.3047911455470276, + "tokens_seen": 3195404288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003408896492728828, + "loss": 2.6695, + "theoretical_loss": 3.3047808561904475, + "tokens_seen": 3195535360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003404619332763045, + "loss": 2.6814, + "theoretical_loss": 3.304770567374065, + "tokens_seen": 3195666432 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003400342172797263, + "loss": 2.5353, + "theoretical_loss": 3.3047602790978305, + "tokens_seen": 3195797504 + }, + { + "epoch": 0.66, + "learning_rate": 0.000339606501283148, + "loss": 2.5045, + "theoretical_loss": 3.304749991361693, + "tokens_seen": 3195928576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003391787852865697, + "loss": 2.5623, + "theoretical_loss": 3.304739704165602, + "tokens_seen": 3196059648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0003387510692899914, + "loss": 2.6663, + "theoretical_loss": 3.304729417509507, + "tokens_seen": 3196190720 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003383233532934132, + "loss": 2.8175, + "theoretical_loss": 3.304719131393358, + "tokens_seen": 3196321792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033789563729683494, + "loss": 2.5404, + "theoretical_loss": 3.304708845817104, + "tokens_seen": 3196452864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033746792130025667, + "loss": 2.5151, + "theoretical_loss": 3.304698560780695, + "tokens_seen": 3196583936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033704020530367834, + "loss": 2.5447, + "theoretical_loss": 3.3046882762840797, + "tokens_seen": 3196715008 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1750966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7685909271240234, + "objective/train/theoretical_loss": 3.3046779923272083, + "objective/train/tokens_used": 226897376, + "theoretical_loss": 3.3046779923272083, + "tokens_seen": 3196846080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033661248930710007, + "loss": 2.6172, + "theoretical_loss": 3.3046779923272083, + "tokens_seen": 3196846080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033618477331052185, + "loss": 2.5337, + "theoretical_loss": 3.30466770891003, + "tokens_seen": 3196977152 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003357570573139436, + "loss": 2.6139, + "theoretical_loss": 3.304657426032495, + "tokens_seen": 3197108224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033532934131736525, + "loss": 2.6706, + "theoretical_loss": 3.304647143694552, + "tokens_seen": 3197239296 + }, + { + "epoch": 0.67, + "learning_rate": 0.000334901625320787, + "loss": 2.5314, + "theoretical_loss": 3.3046368618961512, + "tokens_seen": 3197370368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003344739093242087, + "loss": 2.6955, + "theoretical_loss": 3.3046265806372417, + "tokens_seen": 3197501440 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003340461933276305, + "loss": 2.5646, + "theoretical_loss": 3.3046162999177735, + "tokens_seen": 3197632512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033361847733105217, + "loss": 2.6661, + "theoretical_loss": 3.304606019737696, + "tokens_seen": 3197763584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003331907613344739, + "loss": 2.6769, + "theoretical_loss": 3.304595740096959, + "tokens_seen": 3197894656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033276304533789563, + "loss": 2.5744, + "theoretical_loss": 3.3045854609955114, + "tokens_seen": 3198025728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033233532934131736, + "loss": 2.6286, + "theoretical_loss": 3.304575182433304, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033190761334473914, + "loss": 2.5945, + "theoretical_loss": 3.3045649044102854, + "tokens_seen": 3198287872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003314798973481608, + "loss": 2.6926, + "theoretical_loss": 3.3045546269264054, + "tokens_seen": 3198418944 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1752106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9133894443511963, + "objective/train/theoretical_loss": 3.3045494883866264, + "objective/train/tokens_used": 228535776, + "theoretical_loss": 3.3045494883866264, + "tokens_seen": 3198484480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033105218135158254, + "loss": 2.6499, + "theoretical_loss": 3.304544349981614, + "tokens_seen": 3198550016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033062446535500427, + "loss": 2.5398, + "theoretical_loss": 3.3045340735758604, + "tokens_seen": 3198681088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00033019674935842605, + "loss": 2.7801, + "theoretical_loss": 3.304523797709094, + "tokens_seen": 3198812160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00032976903336184773, + "loss": 2.6759, + "theoretical_loss": 3.3045135223812654, + "tokens_seen": 3198943232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00032934131736526946, + "loss": 2.6202, + "theoretical_loss": 3.304503247592324, + "tokens_seen": 3199074304 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003289136013686912, + "loss": 2.665, + "theoretical_loss": 3.304492973342219, + "tokens_seen": 3199205376 + }, + { + "epoch": 0.67, + "learning_rate": 0.0003284858853721129, + "loss": 2.651, + "theoretical_loss": 3.3044826996309, + "tokens_seen": 3199336448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003280581693755347, + "loss": 2.562, + "theoretical_loss": 3.304472426458317, + "tokens_seen": 3199467520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032763045337895637, + "loss": 2.5694, + "theoretical_loss": 3.30446215382442, + "tokens_seen": 3199598592 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003272027373823781, + "loss": 2.541, + "theoretical_loss": 3.304451881729158, + "tokens_seen": 3199729664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032677502138579983, + "loss": 2.5445, + "theoretical_loss": 3.304441610172481, + "tokens_seen": 3199860736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032634730538922156, + "loss": 2.7194, + "theoretical_loss": 3.304431339154339, + "tokens_seen": 3199991808 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1752559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.096597909927368, + "objective/train/theoretical_loss": 3.304421068674681, + "objective/train/tokens_used": 230174176, + "theoretical_loss": 3.304421068674681, + "tokens_seen": 3200122880 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003259195893926433, + "loss": 2.5557, + "theoretical_loss": 3.304421068674681, + "tokens_seen": 3200122880 + }, + { + "epoch": 0.68, + "learning_rate": 0.000325491873396065, + "loss": 2.501, + "theoretical_loss": 3.3044107987334574, + "tokens_seen": 3200253952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032506415739948674, + "loss": 2.5854, + "theoretical_loss": 3.304400529330618, + "tokens_seen": 3200385024 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003246364414029085, + "loss": 2.6277, + "theoretical_loss": 3.3043902604661115, + "tokens_seen": 3200516096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032420872540633026, + "loss": 2.5145, + "theoretical_loss": 3.3043799921398884, + "tokens_seen": 3200647168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032378100940975193, + "loss": 2.6534, + "theoretical_loss": 3.304369724351899, + "tokens_seen": 3200778240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032335329341317366, + "loss": 2.5208, + "theoretical_loss": 3.3043594571020916, + "tokens_seen": 3200909312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003229255774165954, + "loss": 2.6567, + "theoretical_loss": 3.3043491903904174, + "tokens_seen": 3201040384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003224978614200171, + "loss": 2.7493, + "theoretical_loss": 3.3043389242168253, + "tokens_seen": 3201171456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032207014542343885, + "loss": 2.491, + "theoretical_loss": 3.304328658581265, + "tokens_seen": 3201302528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003216424294268606, + "loss": 2.5666, + "theoretical_loss": 3.304318393483687, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003212147134302823, + "loss": 2.5449, + "theoretical_loss": 3.30430812892404, + "tokens_seen": 3201564672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032078699743370403, + "loss": 2.5378, + "theoretical_loss": 3.304297864902275, + "tokens_seen": 3201695744 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1753714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8973805904388428, + "objective/train/theoretical_loss": 3.3042927330930825, + "objective/train/tokens_used": 231812576, + "theoretical_loss": 3.3042927330930825, + "tokens_seen": 3201761280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00032035928143712576, + "loss": 2.5661, + "theoretical_loss": 3.3042876014183413, + "tokens_seen": 3201826816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003199315654405475, + "loss": 2.4695, + "theoretical_loss": 3.304277338472189, + "tokens_seen": 3201957888 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003195038494439692, + "loss": 2.4664, + "theoretical_loss": 3.3042670760637667, + "tokens_seen": 3202088960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00031907613344739095, + "loss": 2.662, + "theoretical_loss": 3.3042568141930255, + "tokens_seen": 3202220032 + }, + { + "epoch": 0.68, + "learning_rate": 0.0003186484174508127, + "loss": 2.595, + "theoretical_loss": 3.3042465528599148, + "tokens_seen": 3202351104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031822070145423435, + "loss": 2.42, + "theoretical_loss": 3.304236292064384, + "tokens_seen": 3202482176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031779298545765613, + "loss": 2.6321, + "theoretical_loss": 3.304226031806384, + "tokens_seen": 3202613248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031736526946107786, + "loss": 2.4291, + "theoretical_loss": 3.304215772085864, + "tokens_seen": 3202744320 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003169375534644996, + "loss": 2.4285, + "theoretical_loss": 3.304205512902773, + "tokens_seen": 3202875392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031650983746792127, + "loss": 2.542, + "theoretical_loss": 3.3041952542570625, + "tokens_seen": 3203006464 + }, + { + "epoch": 0.69, + "learning_rate": 0.000316082121471343, + "loss": 2.7541, + "theoretical_loss": 3.304184996148681, + "tokens_seen": 3203137536 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003156544054747648, + "loss": 2.5191, + "theoretical_loss": 3.3041747385775793, + "tokens_seen": 3203268608 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1754313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.849808931350708, + "objective/train/theoretical_loss": 3.304164481543707, + "objective/train/tokens_used": 233450976, + "theoretical_loss": 3.304164481543707, + "tokens_seen": 3203399680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003152266894781865, + "loss": 2.6977, + "theoretical_loss": 3.304164481543707, + "tokens_seen": 3203399680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031479897348160823, + "loss": 2.4951, + "theoretical_loss": 3.3041542250470135, + "tokens_seen": 3203530752 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003143712574850299, + "loss": 2.5717, + "theoretical_loss": 3.3041439690874492, + "tokens_seen": 3203661824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003139435414884517, + "loss": 2.5513, + "theoretical_loss": 3.3041337136649642, + "tokens_seen": 3203792896 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003135158254918734, + "loss": 2.659, + "theoretical_loss": 3.3041234587795083, + "tokens_seen": 3203923968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031308810949529515, + "loss": 2.5396, + "theoretical_loss": 3.3041132044310304, + "tokens_seen": 3204055040 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003126603934987168, + "loss": 2.45, + "theoretical_loss": 3.3041029506194817, + "tokens_seen": 3204186112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031223267750213855, + "loss": 2.5278, + "theoretical_loss": 3.3040926973448115, + "tokens_seen": 3204317184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031180496150556034, + "loss": 2.5651, + "theoretical_loss": 3.30408244460697, + "tokens_seen": 3204448256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031137724550898206, + "loss": 2.6935, + "theoretical_loss": 3.304072192405907, + "tokens_seen": 3204579328 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003109495295124038, + "loss": 2.6014, + "theoretical_loss": 3.3040619407415726, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00031052181351582547, + "loss": 2.5716, + "theoretical_loss": 3.304051689613916, + "tokens_seen": 3204841472 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003100940975192472, + "loss": 2.4654, + "theoretical_loss": 3.304041439022888, + "tokens_seen": 3204972544 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1755636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2974143028259277, + "objective/train/theoretical_loss": 3.304036313928594, + "objective/train/tokens_used": 235089376, + "theoretical_loss": 3.304036313928594, + "tokens_seen": 3205038080 + }, + { + "epoch": 0.69, + "learning_rate": 0.000309666381522669, + "loss": 2.5715, + "theoretical_loss": 3.3040311889684384, + "tokens_seen": 3205103616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003092386655260907, + "loss": 2.6104, + "theoretical_loss": 3.3040209394505173, + "tokens_seen": 3205234688 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003088109495295124, + "loss": 2.332, + "theoretical_loss": 3.3040106904690747, + "tokens_seen": 3205365760 + }, + { + "epoch": 0.69, + "learning_rate": 0.0003083832335329341, + "loss": 2.5387, + "theoretical_loss": 3.30400044202406, + "tokens_seen": 3205496832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030795551753635584, + "loss": 2.4564, + "theoretical_loss": 3.303990194115423, + "tokens_seen": 3205627904 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003075278015397776, + "loss": 2.496, + "theoretical_loss": 3.303979946743115, + "tokens_seen": 3205758976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030710008554319935, + "loss": 2.5266, + "theoretical_loss": 3.303969699907085, + "tokens_seen": 3205890048 + }, + { + "epoch": 0.7, + "learning_rate": 0.000306672369546621, + "loss": 2.6383, + "theoretical_loss": 3.303959453607283, + "tokens_seen": 3206021120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030624465355004276, + "loss": 2.5121, + "theoretical_loss": 3.30394920784366, + "tokens_seen": 3206152192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030581693755346454, + "loss": 2.4577, + "theoretical_loss": 3.3039389626161646, + "tokens_seen": 3206283264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030538922155688627, + "loss": 2.5795, + "theoretical_loss": 3.3039287179247476, + "tokens_seen": 3206414336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030496150556030794, + "loss": 2.4733, + "theoretical_loss": 3.303918473769359, + "tokens_seen": 3206545408 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1756084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.320000171661377, + "objective/train/theoretical_loss": 3.303908230149949, + "objective/train/tokens_used": 236727776, + "theoretical_loss": 3.303908230149949, + "tokens_seen": 3206676480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030453378956372967, + "loss": 2.6419, + "theoretical_loss": 3.303908230149949, + "tokens_seen": 3206676480 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003041060735671514, + "loss": 2.5157, + "theoretical_loss": 3.3038979870664678, + "tokens_seen": 3206807552 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003036783575705732, + "loss": 2.5942, + "theoretical_loss": 3.303887744518865, + "tokens_seen": 3206938624 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003032506415739949, + "loss": 2.551, + "theoretical_loss": 3.30387750250709, + "tokens_seen": 3207069696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003028229255774166, + "loss": 2.485, + "theoretical_loss": 3.303867261031094, + "tokens_seen": 3207200768 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003023952095808383, + "loss": 2.6017, + "theoretical_loss": 3.303857020090827, + "tokens_seen": 3207331840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030196749358426004, + "loss": 2.5721, + "theoretical_loss": 3.3038467796862387, + "tokens_seen": 3207462912 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003015397775876818, + "loss": 2.5352, + "theoretical_loss": 3.3038365398172793, + "tokens_seen": 3207593984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0003011120615911035, + "loss": 2.6202, + "theoretical_loss": 3.303826300483899, + "tokens_seen": 3207725056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030068434559452523, + "loss": 2.5636, + "theoretical_loss": 3.3038160616860477, + "tokens_seen": 3207856128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00030025662959794696, + "loss": 2.4952, + "theoretical_loss": 3.3038058234236756, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002998289136013687, + "loss": 2.5859, + "theoretical_loss": 3.303795585696733, + "tokens_seen": 3208118272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002994011976047904, + "loss": 2.5651, + "theoretical_loss": 3.30378534850517, + "tokens_seen": 3208249344 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1757266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.609693765640259, + "objective/train/theoretical_loss": 3.30378023011014, + "objective/train/tokens_used": 238366176, + "theoretical_loss": 3.30378023011014, + "tokens_seen": 3208314880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00029897348160821214, + "loss": 2.5566, + "theoretical_loss": 3.3037751118489362, + "tokens_seen": 3208380416 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002985457656116339, + "loss": 2.6598, + "theoretical_loss": 3.3037648757279827, + "tokens_seen": 3208511488 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002981180496150556, + "loss": 2.4844, + "theoretical_loss": 3.3037546401422584, + "tokens_seen": 3208642560 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002976903336184774, + "loss": 2.3874, + "theoretical_loss": 3.3037444050917144, + "tokens_seen": 3208773632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029726261762189906, + "loss": 2.5498, + "theoretical_loss": 3.303734170576301, + "tokens_seen": 3208904704 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002968349016253208, + "loss": 2.7152, + "theoretical_loss": 3.3037239365959676, + "tokens_seen": 3209035776 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002964071856287425, + "loss": 2.604, + "theoretical_loss": 3.3037137031506645, + "tokens_seen": 3209166848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029597946963216425, + "loss": 2.5602, + "theoretical_loss": 3.3037034702403423, + "tokens_seen": 3209297920 + }, + { + "epoch": 0.71, + "learning_rate": 0.000295551753635586, + "loss": 2.5005, + "theoretical_loss": 3.303693237864951, + "tokens_seen": 3209428992 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002951240376390077, + "loss": 2.5691, + "theoretical_loss": 3.303683006024441, + "tokens_seen": 3209560064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029469632164242943, + "loss": 2.6238, + "theoretical_loss": 3.3036727747187618, + "tokens_seen": 3209691136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029426860564585116, + "loss": 2.5866, + "theoretical_loss": 3.303662543947864, + "tokens_seen": 3209822208 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1757623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.177359104156494, + "objective/train/theoretical_loss": 3.303652313711699, + "objective/train/tokens_used": 240004576, + "theoretical_loss": 3.303652313711699, + "tokens_seen": 3209953280 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002938408896492729, + "loss": 2.6727, + "theoretical_loss": 3.303652313711699, + "tokens_seen": 3209953280 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002934131736526946, + "loss": 2.5815, + "theoretical_loss": 3.3036420840102148, + "tokens_seen": 3210084352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029298545765611635, + "loss": 2.573, + "theoretical_loss": 3.3036318548433634, + "tokens_seen": 3210215424 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002925577416595381, + "loss": 2.4612, + "theoretical_loss": 3.303621626211094, + "tokens_seen": 3210346496 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002921300256629598, + "loss": 2.4738, + "theoretical_loss": 3.303611398113357, + "tokens_seen": 3210477568 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002917023096663815, + "loss": 2.71, + "theoretical_loss": 3.3036011705501034, + "tokens_seen": 3210608640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00029127459366980326, + "loss": 2.5822, + "theoretical_loss": 3.3035909435212827, + "tokens_seen": 3210739712 + }, + { + "epoch": 0.71, + "learning_rate": 0.000290846877673225, + "loss": 2.5685, + "theoretical_loss": 3.303580717026845, + "tokens_seen": 3210870784 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002904191616766467, + "loss": 2.6034, + "theoretical_loss": 3.3035704910667416, + "tokens_seen": 3211001856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00028999144568006845, + "loss": 2.4281, + "theoretical_loss": 3.3035602656409218, + "tokens_seen": 3211132928 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002895637296834902, + "loss": 2.7199, + "theoretical_loss": 3.303550040749336, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002891360136869119, + "loss": 2.5815, + "theoretical_loss": 3.3035398163919347, + "tokens_seen": 3211395072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00028870829769033363, + "loss": 2.7606, + "theoretical_loss": 3.303529592568668, + "tokens_seen": 3211526144 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1758775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5950331687927246, + "objective/train/theoretical_loss": 3.3035244808573196, + "objective/train/tokens_used": 241642976, + "theoretical_loss": 3.3035244808573196, + "tokens_seen": 3211591680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00028828058169375536, + "loss": 2.5859, + "theoretical_loss": 3.3035193692794866, + "tokens_seen": 3211657216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028785286569717704, + "loss": 2.494, + "theoretical_loss": 3.3035091465243407, + "tokens_seen": 3211788288 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002874251497005988, + "loss": 2.4989, + "theoretical_loss": 3.3034989243031805, + "tokens_seen": 3211919360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028699743370402055, + "loss": 2.3876, + "theoretical_loss": 3.303488702615956, + "tokens_seen": 3212050432 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002865697177074423, + "loss": 2.4877, + "theoretical_loss": 3.303478481462618, + "tokens_seen": 3212181504 + }, + { + "epoch": 0.72, + "learning_rate": 0.000286142001710864, + "loss": 2.6349, + "theoretical_loss": 3.3034682608431165, + "tokens_seen": 3212312576 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002857142857142857, + "loss": 2.4448, + "theoretical_loss": 3.303458040757402, + "tokens_seen": 3212443648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028528656971770746, + "loss": 2.599, + "theoretical_loss": 3.303447821205425, + "tokens_seen": 3212574720 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002848588537211292, + "loss": 2.4383, + "theoretical_loss": 3.3034376021871354, + "tokens_seen": 3212705792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002844311377245509, + "loss": 2.5213, + "theoretical_loss": 3.303427383702484, + "tokens_seen": 3212836864 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002840034217279726, + "loss": 2.4449, + "theoretical_loss": 3.303417165751421, + "tokens_seen": 3212967936 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002835757057313943, + "loss": 2.6495, + "theoretical_loss": 3.3034069483338966, + "tokens_seen": 3213099008 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1759176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.307115077972412, + "objective/train/theoretical_loss": 3.303396731449862, + "objective/train/tokens_used": 243281376, + "theoretical_loss": 3.303396731449862, + "tokens_seen": 3213230080 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002831479897348161, + "loss": 2.5473, + "theoretical_loss": 3.303396731449862, + "tokens_seen": 3213230080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028272027373823784, + "loss": 2.3969, + "theoretical_loss": 3.303386515099266, + "tokens_seen": 3213361152 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002822925577416595, + "loss": 2.6036, + "theoretical_loss": 3.3033762992820606, + "tokens_seen": 3213492224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028186484174508124, + "loss": 2.6279, + "theoretical_loss": 3.303366083998195, + "tokens_seen": 3213623296 + }, + { + "epoch": 0.72, + "learning_rate": 0.000281437125748503, + "loss": 2.5705, + "theoretical_loss": 3.3033558692476204, + "tokens_seen": 3213754368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028100940975192475, + "loss": 2.656, + "theoretical_loss": 3.303345655030287, + "tokens_seen": 3213885440 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002805816937553465, + "loss": 2.6342, + "theoretical_loss": 3.303335441346145, + "tokens_seen": 3214016512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00028015397775876816, + "loss": 2.5266, + "theoretical_loss": 3.3033252281951455, + "tokens_seen": 3214147584 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002797262617621899, + "loss": 2.4818, + "theoretical_loss": 3.3033150155772377, + "tokens_seen": 3214278656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00027929854576561167, + "loss": 2.5873, + "theoretical_loss": 3.303304803492373, + "tokens_seen": 3214409728 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002788708297690334, + "loss": 2.6208, + "theoretical_loss": 3.3032945919405017, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00027844311377245507, + "loss": 2.5698, + "theoretical_loss": 3.303284380921574, + "tokens_seen": 3214671872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002780153977758768, + "loss": 2.5978, + "theoretical_loss": 3.303274170435541, + "tokens_seen": 3214802944 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1760371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.48292875289917, + "objective/train/theoretical_loss": 3.3032690653923438, + "objective/train/tokens_used": 244919776, + "theoretical_loss": 3.3032690653923438, + "tokens_seen": 3214868480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027758768177929853, + "loss": 2.4452, + "theoretical_loss": 3.3032639604823517, + "tokens_seen": 3214934016 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002771599657827203, + "loss": 2.3875, + "theoretical_loss": 3.303253751061958, + "tokens_seen": 3215065088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027673224978614204, + "loss": 2.4726, + "theoretical_loss": 3.3032435421743105, + "tokens_seen": 3215196160 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002763045337895637, + "loss": 2.5305, + "theoretical_loss": 3.3032333338193585, + "tokens_seen": 3215327232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027587681779298544, + "loss": 2.6058, + "theoretical_loss": 3.3032231259970533, + "tokens_seen": 3215458304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027544910179640717, + "loss": 2.6213, + "theoretical_loss": 3.303212918707345, + "tokens_seen": 3215589376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027502138579982895, + "loss": 2.5544, + "theoretical_loss": 3.3032027119501843, + "tokens_seen": 3215720448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027459366980325063, + "loss": 2.586, + "theoretical_loss": 3.303192505725522, + "tokens_seen": 3215851520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027416595380667236, + "loss": 2.4732, + "theoretical_loss": 3.303182300033308, + "tokens_seen": 3215982592 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002737382378100941, + "loss": 2.4938, + "theoretical_loss": 3.303172094873493, + "tokens_seen": 3216113664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027331052181351587, + "loss": 2.5529, + "theoretical_loss": 3.3031618902460282, + "tokens_seen": 3216244736 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002728828058169376, + "loss": 2.6591, + "theoretical_loss": 3.303151686150863, + "tokens_seen": 3216375808 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1760932, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7813944816589355, + "objective/train/theoretical_loss": 3.303141482587949, + "objective/train/tokens_used": 246558176, + "theoretical_loss": 3.303141482587949, + "tokens_seen": 3216506880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027245508982035927, + "loss": 2.5861, + "theoretical_loss": 3.303141482587949, + "tokens_seen": 3216506880 + }, + { + "epoch": 0.73, + "learning_rate": 0.000272027373823781, + "loss": 2.6047, + "theoretical_loss": 3.303131279557236, + "tokens_seen": 3216637952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00027159965782720273, + "loss": 2.544, + "theoretical_loss": 3.303121077058675, + "tokens_seen": 3216769024 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002711719418306245, + "loss": 2.6757, + "theoretical_loss": 3.3031108750922167, + "tokens_seen": 3216900096 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002707442258340462, + "loss": 2.4401, + "theoretical_loss": 3.303100673657811, + "tokens_seen": 3217031168 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002703165098374679, + "loss": 2.5899, + "theoretical_loss": 3.303090472755409, + "tokens_seen": 3217162240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00026988879384088965, + "loss": 2.6545, + "theoretical_loss": 3.3030802723849613, + "tokens_seen": 3217293312 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002694610778443114, + "loss": 2.4423, + "theoretical_loss": 3.3030700725464177, + "tokens_seen": 3217424384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00026903336184773316, + "loss": 2.5879, + "theoretical_loss": 3.30305987323973, + "tokens_seen": 3217555456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00026860564585115483, + "loss": 2.486, + "theoretical_loss": 3.303049674464848, + "tokens_seen": 3217686528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00026817792985457656, + "loss": 2.5141, + "theoretical_loss": 3.303039476221723, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002677502138579983, + "loss": 2.587, + "theoretical_loss": 3.303029278510305, + "tokens_seen": 3217948672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026732249786142, + "loss": 2.394, + "theoretical_loss": 3.3030190813305444, + "tokens_seen": 3218079744 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1762106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7778732776641846, + "objective/train/theoretical_loss": 3.3030139829400205, + "objective/train/tokens_used": 248196576, + "theoretical_loss": 3.3030139829400205, + "tokens_seen": 3218145280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026689478186484175, + "loss": 2.6011, + "theoretical_loss": 3.3030088846823924, + "tokens_seen": 3218210816 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002664670658682635, + "loss": 2.6277, + "theoretical_loss": 3.3029986885657996, + "tokens_seen": 3218341888 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002660393498716852, + "loss": 2.6436, + "theoretical_loss": 3.3029884929807163, + "tokens_seen": 3218472960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026561163387510693, + "loss": 2.4285, + "theoretical_loss": 3.3029782979270936, + "tokens_seen": 3218604032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026518391787852866, + "loss": 2.3964, + "theoretical_loss": 3.302968103404882, + "tokens_seen": 3218735104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002647562018819504, + "loss": 2.6014, + "theoretical_loss": 3.302957909414032, + "tokens_seen": 3218866176 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002643284858853721, + "loss": 2.5703, + "theoretical_loss": 3.3029477159544935, + "tokens_seen": 3218997248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026390076988879385, + "loss": 2.4542, + "theoretical_loss": 3.302937523026219, + "tokens_seen": 3219128320 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002634730538922156, + "loss": 2.4293, + "theoretical_loss": 3.3029273306291578, + "tokens_seen": 3219259392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002630453378956373, + "loss": 2.5482, + "theoretical_loss": 3.302917138763261, + "tokens_seen": 3219390464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026261762189905903, + "loss": 2.4836, + "theoretical_loss": 3.3029069474284793, + "tokens_seen": 3219521536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026218990590248076, + "loss": 2.4569, + "theoretical_loss": 3.3028967566247633, + "tokens_seen": 3219652608 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1762646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6974568367004395, + "objective/train/theoretical_loss": 3.302886566352064, + "objective/train/tokens_used": 249834976, + "theoretical_loss": 3.302886566352064, + "tokens_seen": 3219783680 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002617621899059025, + "loss": 2.4433, + "theoretical_loss": 3.302886566352064, + "tokens_seen": 3219783680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026133447390932417, + "loss": 2.625, + "theoretical_loss": 3.3028763766103317, + "tokens_seen": 3219914752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00026090675791274595, + "loss": 2.5281, + "theoretical_loss": 3.3028661873995175, + "tokens_seen": 3220045824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002604790419161677, + "loss": 2.5798, + "theoretical_loss": 3.302855998719572, + "tokens_seen": 3220176896 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002600513259195894, + "loss": 2.561, + "theoretical_loss": 3.3028458105704455, + "tokens_seen": 3220307968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00025962360992301114, + "loss": 2.4877, + "theoretical_loss": 3.3028356229520894, + "tokens_seen": 3220439040 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002591958939264328, + "loss": 2.4978, + "theoretical_loss": 3.302825435864454, + "tokens_seen": 3220570112 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002587681779298546, + "loss": 2.5616, + "theoretical_loss": 3.3028152493074905, + "tokens_seen": 3220701184 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002583404619332763, + "loss": 2.5324, + "theoretical_loss": 3.3028050632811494, + "tokens_seen": 3220832256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00025791274593669805, + "loss": 2.51, + "theoretical_loss": 3.302794877785381, + "tokens_seen": 3220963328 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002574850299401197, + "loss": 2.5464, + "theoretical_loss": 3.3027846928201368, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002570573139435415, + "loss": 2.6245, + "theoretical_loss": 3.3027745083853675, + "tokens_seen": 3221225472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025662959794696324, + "loss": 2.6352, + "theoretical_loss": 3.302764324481023, + "tokens_seen": 3221356544 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1763846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4329378604888916, + "objective/train/theoretical_loss": 3.3027592327277455, + "objective/train/tokens_used": 251473376, + "theoretical_loss": 3.3027592327277455, + "tokens_seen": 3221422080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025620188195038497, + "loss": 2.4117, + "theoretical_loss": 3.302754141107056, + "tokens_seen": 3221487616 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002557741659538067, + "loss": 2.4146, + "theoretical_loss": 3.3027439582634153, + "tokens_seen": 3221618688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025534644995722837, + "loss": 2.5075, + "theoretical_loss": 3.3027337759500526, + "tokens_seen": 3221749760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025491873396065015, + "loss": 2.4822, + "theoretical_loss": 3.3027235941669186, + "tokens_seen": 3221880832 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002544910179640719, + "loss": 2.337, + "theoretical_loss": 3.302713412913964, + "tokens_seen": 3222011904 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002540633019674936, + "loss": 2.4339, + "theoretical_loss": 3.3027032321911403, + "tokens_seen": 3222142976 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002536355859709153, + "loss": 2.5068, + "theoretical_loss": 3.3026930519983972, + "tokens_seen": 3222274048 + }, + { + "epoch": 0.75, + "learning_rate": 0.000253207869974337, + "loss": 2.4265, + "theoretical_loss": 3.3026828723356862, + "tokens_seen": 3222405120 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002527801539777588, + "loss": 2.5125, + "theoretical_loss": 3.3026726932029584, + "tokens_seen": 3222536192 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002523524379811805, + "loss": 2.4566, + "theoretical_loss": 3.302662514600164, + "tokens_seen": 3222667264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025192472198460225, + "loss": 2.5294, + "theoretical_loss": 3.302652336527254, + "tokens_seen": 3222798336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025149700598802393, + "loss": 2.4201, + "theoretical_loss": 3.30264215898418, + "tokens_seen": 3222929408 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1764511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.071906805038452, + "objective/train/theoretical_loss": 3.302631981970892, + "objective/train/tokens_used": 253111776, + "theoretical_loss": 3.302631981970892, + "tokens_seen": 3223060480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025106928999144566, + "loss": 2.6362, + "theoretical_loss": 3.302631981970892, + "tokens_seen": 3223060480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025064157399486744, + "loss": 2.6805, + "theoretical_loss": 3.3026218054873415, + "tokens_seen": 3223191552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00025021385799828917, + "loss": 2.4625, + "theoretical_loss": 3.302611629533479, + "tokens_seen": 3223322624 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002497861420017109, + "loss": 2.5851, + "theoretical_loss": 3.302601454109255, + "tokens_seen": 3223453696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00024935842600513257, + "loss": 2.4232, + "theoretical_loss": 3.3025912792146217, + "tokens_seen": 3223584768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00024893071000855435, + "loss": 2.4813, + "theoretical_loss": 3.302581104849529, + "tokens_seen": 3223715840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00024850299401197603, + "loss": 2.492, + "theoretical_loss": 3.3025709310139275, + "tokens_seen": 3223846912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00024807527801539776, + "loss": 2.527, + "theoretical_loss": 3.302560757707769, + "tokens_seen": 3223977984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00024764756201881954, + "loss": 2.5057, + "theoretical_loss": 3.302550584931004, + "tokens_seen": 3224109056 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002472198460222412, + "loss": 2.4994, + "theoretical_loss": 3.302540412683584, + "tokens_seen": 3224240128 + }, + { + "epoch": 0.76, + "learning_rate": 0.000246792130025663, + "loss": 2.5004, + "theoretical_loss": 3.302530240965458, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024636441402908467, + "loss": 2.5225, + "theoretical_loss": 3.3025200697765795, + "tokens_seen": 3224502272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024593669803250646, + "loss": 2.4992, + "theoretical_loss": 3.302509899116898, + "tokens_seen": 3224633344 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1765173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8799469470977783, + "objective/train/theoretical_loss": 3.3025048139854913, + "objective/train/tokens_used": 254750176, + "theoretical_loss": 3.3025048139854913, + "tokens_seen": 3224698880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024550898203592813, + "loss": 2.5203, + "theoretical_loss": 3.302499728986365, + "tokens_seen": 3224764416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024508126603934986, + "loss": 2.5546, + "theoretical_loss": 3.302489559384931, + "tokens_seen": 3224895488 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002446535500427716, + "loss": 2.4764, + "theoretical_loss": 3.3024793903125476, + "tokens_seen": 3225026560 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002442258340461933, + "loss": 2.5792, + "theoretical_loss": 3.3024692217691647, + "tokens_seen": 3225157632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024379811804961505, + "loss": 2.624, + "theoretical_loss": 3.3024590537547343, + "tokens_seen": 3225288704 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002433704020530368, + "loss": 2.5896, + "theoretical_loss": 3.302448886269207, + "tokens_seen": 3225419776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024294268605645853, + "loss": 2.7106, + "theoretical_loss": 3.302438719312534, + "tokens_seen": 3225550848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024251497005988023, + "loss": 2.5738, + "theoretical_loss": 3.302428552884666, + "tokens_seen": 3225681920 + }, + { + "epoch": 0.76, + "learning_rate": 0.000242087254063302, + "loss": 2.4941, + "theoretical_loss": 3.302418386985554, + "tokens_seen": 3225812992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002416595380667237, + "loss": 2.4776, + "theoretical_loss": 3.3024082216151496, + "tokens_seen": 3225944064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024123182207014544, + "loss": 2.7235, + "theoretical_loss": 3.3023980567734035, + "tokens_seen": 3226075136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024080410607356715, + "loss": 2.2927, + "theoretical_loss": 3.302387892460266, + "tokens_seen": 3226206208 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1765556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7151641845703125, + "objective/train/theoretical_loss": 3.3023777286756895, + "objective/train/tokens_used": 256388576, + "theoretical_loss": 3.3023777286756895, + "tokens_seen": 3226337280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00024037639007698888, + "loss": 2.5894, + "theoretical_loss": 3.3023777286756895, + "tokens_seen": 3226337280 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002399486740804106, + "loss": 2.5522, + "theoretical_loss": 3.302367565419624, + "tokens_seen": 3226468352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00023952095808383233, + "loss": 2.6926, + "theoretical_loss": 3.302357402692021, + "tokens_seen": 3226599424 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002390932420872541, + "loss": 2.6711, + "theoretical_loss": 3.3023472404928307, + "tokens_seen": 3226730496 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002386655260906758, + "loss": 2.6496, + "theoretical_loss": 3.3023370788220054, + "tokens_seen": 3226861568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00023823781009409755, + "loss": 2.5713, + "theoretical_loss": 3.302326917679496, + "tokens_seen": 3226992640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00023781009409751925, + "loss": 2.5756, + "theoretical_loss": 3.3023167570652525, + "tokens_seen": 3227123712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023738237810094098, + "loss": 2.5765, + "theoretical_loss": 3.302306596979227, + "tokens_seen": 3227254784 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002369546621043627, + "loss": 2.6775, + "theoretical_loss": 3.302296437421371, + "tokens_seen": 3227385856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023652694610778443, + "loss": 2.5556, + "theoretical_loss": 3.302286278391634, + "tokens_seen": 3227516928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023609923011120616, + "loss": 2.6363, + "theoretical_loss": 3.302276119889968, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002356715141146279, + "loss": 2.6966, + "theoretical_loss": 3.3022659619163246, + "tokens_seen": 3227779072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023524379811804962, + "loss": 2.5517, + "theoretical_loss": 3.3022558044706543, + "tokens_seen": 3227910144 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1765973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5320229530334473, + "objective/train/theoretical_loss": 3.3022507259457936, + "objective/train/tokens_used": 258026976, + "theoretical_loss": 3.3022507259457936, + "tokens_seen": 3227975680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023481608212147135, + "loss": 2.5281, + "theoretical_loss": 3.3022456475529083, + "tokens_seen": 3228041216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023438836612489308, + "loss": 2.5274, + "theoretical_loss": 3.3022354911630374, + "tokens_seen": 3228172288 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002339606501283148, + "loss": 2.6598, + "theoretical_loss": 3.3022253353009936, + "tokens_seen": 3228303360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023353293413173654, + "loss": 2.5662, + "theoretical_loss": 3.302215179966727, + "tokens_seen": 3228434432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023310521813515826, + "loss": 2.4828, + "theoretical_loss": 3.3022050251601898, + "tokens_seen": 3228565504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023267750213858, + "loss": 2.488, + "theoretical_loss": 3.3021948708813325, + "tokens_seen": 3228696576 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002322497861420017, + "loss": 2.5845, + "theoretical_loss": 3.302184717130106, + "tokens_seen": 3228827648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023182207014542345, + "loss": 2.5981, + "theoretical_loss": 3.3021745639064624, + "tokens_seen": 3228958720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023139435414884515, + "loss": 2.6673, + "theoretical_loss": 3.3021644112103523, + "tokens_seen": 3229089792 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002309666381522669, + "loss": 2.7423, + "theoretical_loss": 3.3021542590417265, + "tokens_seen": 3229220864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023053892215568864, + "loss": 2.563, + "theoretical_loss": 3.302144107400537, + "tokens_seen": 3229351936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00023011120615911037, + "loss": 2.4334, + "theoretical_loss": 3.3021339562867342, + "tokens_seen": 3229483008 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1767178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4786245822906494, + "objective/train/theoretical_loss": 3.30212380570027, + "objective/train/tokens_used": 259665376, + "theoretical_loss": 3.30212380570027, + "tokens_seen": 3229614080 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002296834901625321, + "loss": 2.5714, + "theoretical_loss": 3.30212380570027, + "tokens_seen": 3229614080 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002292557741659538, + "loss": 2.5436, + "theoretical_loss": 3.3021136556410955, + "tokens_seen": 3229745152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00022882805816937555, + "loss": 2.6152, + "theoretical_loss": 3.3021035061091615, + "tokens_seen": 3229876224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00022840034217279725, + "loss": 2.7522, + "theoretical_loss": 3.302093357104419, + "tokens_seen": 3230007296 + }, + { + "epoch": 0.77, + "learning_rate": 0.000227972626176219, + "loss": 2.5851, + "theoretical_loss": 3.3020832086268204, + "tokens_seen": 3230138368 + }, + { + "epoch": 0.77, + "learning_rate": 0.0002275449101796407, + "loss": 2.4995, + "theoretical_loss": 3.3020730606763156, + "tokens_seen": 3230269440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022711719418306247, + "loss": 2.5256, + "theoretical_loss": 3.3020629132528567, + "tokens_seen": 3230400512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022668947818648417, + "loss": 2.6279, + "theoretical_loss": 3.3020527663563946, + "tokens_seen": 3230531584 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002262617621899059, + "loss": 2.5888, + "theoretical_loss": 3.3020426199868806, + "tokens_seen": 3230662656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022583404619332765, + "loss": 2.6261, + "theoretical_loss": 3.3020324741442657, + "tokens_seen": 3230793728 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022540633019674935, + "loss": 2.553, + "theoretical_loss": 3.3020223288285018, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002249786142001711, + "loss": 2.4358, + "theoretical_loss": 3.3020121840395396, + "tokens_seen": 3231055872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002245508982035928, + "loss": 2.5213, + "theoretical_loss": 3.3020020397773306, + "tokens_seen": 3231186944 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1767743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5154170989990234, + "objective/train/theoretical_loss": 3.3019969678437437, + "objective/train/tokens_used": 261303776, + "theoretical_loss": 3.3019969678437437, + "tokens_seen": 3231252480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022412318220701454, + "loss": 2.6154, + "theoretical_loss": 3.3019918960418266, + "tokens_seen": 3231318016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022369546621043627, + "loss": 2.6876, + "theoretical_loss": 3.301981752832978, + "tokens_seen": 3231449088 + }, + { + "epoch": 0.78, + "learning_rate": 0.000223267750213858, + "loss": 2.6769, + "theoretical_loss": 3.301971610150736, + "tokens_seen": 3231580160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022284003421727973, + "loss": 2.5933, + "theoretical_loss": 3.301961467995053, + "tokens_seen": 3231711232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022241231822070146, + "loss": 2.6092, + "theoretical_loss": 3.301951326365879, + "tokens_seen": 3231842304 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002219846022241232, + "loss": 2.4552, + "theoretical_loss": 3.301941185263167, + "tokens_seen": 3231973376 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002215568862275449, + "loss": 2.4722, + "theoretical_loss": 3.3019310446868664, + "tokens_seen": 3232104448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022112917023096664, + "loss": 2.5452, + "theoretical_loss": 3.3019209046369298, + "tokens_seen": 3232235520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00022070145423438837, + "loss": 2.6044, + "theoretical_loss": 3.301910765113308, + "tokens_seen": 3232366592 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002202737382378101, + "loss": 2.51, + "theoretical_loss": 3.3019006261159527, + "tokens_seen": 3232497664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021984602224123183, + "loss": 2.5195, + "theoretical_loss": 3.301890487644815, + "tokens_seen": 3232628736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021941830624465356, + "loss": 2.5523, + "theoretical_loss": 3.301880349699846, + "tokens_seen": 3232759808 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1769062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2536916732788086, + "objective/train/theoretical_loss": 3.3018702122809978, + "objective/train/tokens_used": 262942176, + "theoretical_loss": 3.3018702122809978, + "tokens_seen": 3232890880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021899059024807529, + "loss": 2.4897, + "theoretical_loss": 3.3018702122809978, + "tokens_seen": 3232890880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021856287425149701, + "loss": 2.474, + "theoretical_loss": 3.3018600753882215, + "tokens_seen": 3233021952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021813515825491872, + "loss": 2.4799, + "theoretical_loss": 3.3018499390214675, + "tokens_seen": 3233153024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00021770744225834047, + "loss": 2.4787, + "theoretical_loss": 3.3018398031806884, + "tokens_seen": 3233284096 + }, + { + "epoch": 0.78, + "learning_rate": 0.0002172797262617622, + "loss": 2.3919, + "theoretical_loss": 3.3018296678658356, + "tokens_seen": 3233415168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021685201026518393, + "loss": 2.6312, + "theoretical_loss": 3.3018195330768596, + "tokens_seen": 3233546240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021642429426860566, + "loss": 2.5643, + "theoretical_loss": 3.3018093988137123, + "tokens_seen": 3233677312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021599657827202736, + "loss": 2.6163, + "theoretical_loss": 3.3017992650763452, + "tokens_seen": 3233808384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021556886227544912, + "loss": 2.6987, + "theoretical_loss": 3.3017891318647097, + "tokens_seen": 3233939456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021514114627887082, + "loss": 2.7174, + "theoretical_loss": 3.301778999178757, + "tokens_seen": 3234070528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021471343028229257, + "loss": 2.6426, + "theoretical_loss": 3.3017688670184384, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021428571428571427, + "loss": 2.5005, + "theoretical_loss": 3.301758735383706, + "tokens_seen": 3234332672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021385799828913603, + "loss": 2.5663, + "theoretical_loss": 3.301748604274511, + "tokens_seen": 3234463744 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1769583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8406972885131836, + "objective/train/theoretical_loss": 3.3017435389169743, + "objective/train/tokens_used": 264580576, + "theoretical_loss": 3.3017435389169743, + "tokens_seen": 3234529280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021343028229255776, + "loss": 2.5101, + "theoretical_loss": 3.3017384736908038, + "tokens_seen": 3234594816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021300256629597946, + "loss": 2.4787, + "theoretical_loss": 3.3017283436325373, + "tokens_seen": 3234725888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021257485029940122, + "loss": 2.4298, + "theoretical_loss": 3.301718214099662, + "tokens_seen": 3234856960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021214713430282292, + "loss": 2.5347, + "theoretical_loss": 3.3017080850921303, + "tokens_seen": 3234988032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021171941830624467, + "loss": 2.4556, + "theoretical_loss": 3.3016979566098925, + "tokens_seen": 3235119104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021129170230966638, + "loss": 2.7056, + "theoretical_loss": 3.301687828652901, + "tokens_seen": 3235250176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021086398631308813, + "loss": 2.5703, + "theoretical_loss": 3.3016777012211067, + "tokens_seen": 3235381248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021043627031650983, + "loss": 2.4976, + "theoretical_loss": 3.301667574314462, + "tokens_seen": 3235512320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00021000855431993156, + "loss": 2.6309, + "theoretical_loss": 3.301657447932917, + "tokens_seen": 3235643392 + }, + { + "epoch": 0.79, + "learning_rate": 0.0002095808383233533, + "loss": 2.7144, + "theoretical_loss": 3.301647322076424, + "tokens_seen": 3235774464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00020915312232677502, + "loss": 2.6534, + "theoretical_loss": 3.3016371967449345, + "tokens_seen": 3235905536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00020872540633019678, + "loss": 2.5027, + "theoretical_loss": 3.3016270719384004, + "tokens_seen": 3236036608 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1770972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1470959186553955, + "objective/train/theoretical_loss": 3.3016169476567727, + "objective/train/tokens_used": 266218976, + "theoretical_loss": 3.3016169476567727, + "tokens_seen": 3236167680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00020829769033361848, + "loss": 2.5309, + "theoretical_loss": 3.3016169476567727, + "tokens_seen": 3236167680 + }, + { + "epoch": 0.79, + "learning_rate": 0.0002078699743370402, + "loss": 2.5764, + "theoretical_loss": 3.301606823900003, + "tokens_seen": 3236298752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00020744225834046193, + "loss": 2.5439, + "theoretical_loss": 3.3015967006680422, + "tokens_seen": 3236429824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020701454234388366, + "loss": 2.3709, + "theoretical_loss": 3.301586577960843, + "tokens_seen": 3236560896 + }, + { + "epoch": 0.8, + "learning_rate": 0.0002065868263473054, + "loss": 2.4866, + "theoretical_loss": 3.3015764557783567, + "tokens_seen": 3236691968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020615911035072712, + "loss": 2.6296, + "theoretical_loss": 3.301566334120534, + "tokens_seen": 3236823040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020573139435414885, + "loss": 2.7243, + "theoretical_loss": 3.301556212987328, + "tokens_seen": 3236954112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020530367835757058, + "loss": 2.5835, + "theoretical_loss": 3.3015460923786883, + "tokens_seen": 3237085184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0002048759623609923, + "loss": 2.5565, + "theoretical_loss": 3.301535972294568, + "tokens_seen": 3237216256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020444824636441404, + "loss": 2.4643, + "theoretical_loss": 3.301525852734918, + "tokens_seen": 3237347328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020402053036783576, + "loss": 2.5406, + "theoretical_loss": 3.3015157336996905, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.8, + "learning_rate": 0.0002035928143712575, + "loss": 2.5859, + "theoretical_loss": 3.3015056151888365, + "tokens_seen": 3237609472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020316509837467922, + "loss": 2.5665, + "theoretical_loss": 3.3014954972023074, + "tokens_seen": 3237740544 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1771594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3671107292175293, + "objective/train/theoretical_loss": 3.30149043840565, + "objective/train/tokens_used": 267857376, + "theoretical_loss": 3.30149043840565, + "tokens_seen": 3237806080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020273738237810092, + "loss": 2.6425, + "theoretical_loss": 3.3014853797400554, + "tokens_seen": 3237871616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020230966638152268, + "loss": 2.6033, + "theoretical_loss": 3.301475262802032, + "tokens_seen": 3238002688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020188195038494438, + "loss": 2.5792, + "theoretical_loss": 3.3014651463881886, + "tokens_seen": 3238133760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020145423438836614, + "loss": 2.5413, + "theoretical_loss": 3.301455030498477, + "tokens_seen": 3238264832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020102651839178784, + "loss": 2.5428, + "theoretical_loss": 3.3014449151328487, + "tokens_seen": 3238395904 + }, + { + "epoch": 0.8, + "learning_rate": 0.0002005988023952096, + "loss": 2.4654, + "theoretical_loss": 3.301434800291255, + "tokens_seen": 3238526976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00020017108639863132, + "loss": 2.4941, + "theoretical_loss": 3.3014246859736485, + "tokens_seen": 3238658048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00019974337040205303, + "loss": 2.5441, + "theoretical_loss": 3.30141457217998, + "tokens_seen": 3238789120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00019931565440547478, + "loss": 2.4707, + "theoretical_loss": 3.3014044589102016, + "tokens_seen": 3238920192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00019888793840889648, + "loss": 2.5503, + "theoretical_loss": 3.3013943461642645, + "tokens_seen": 3239051264 + }, + { + "epoch": 0.8, + "learning_rate": 0.00019846022241231824, + "loss": 2.6467, + "theoretical_loss": 3.3013842339421213, + "tokens_seen": 3239182336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00019803250641573994, + "loss": 2.5602, + "theoretical_loss": 3.3013741222437227, + "tokens_seen": 3239313408 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1772760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.39023756980896, + "objective/train/theoretical_loss": 3.3013640110690208, + "objective/train/tokens_used": 269495776, + "theoretical_loss": 3.3013640110690208, + "tokens_seen": 3239444480 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001976047904191617, + "loss": 2.454, + "theoretical_loss": 3.3013640110690208, + "tokens_seen": 3239444480 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001971770744225834, + "loss": 2.3576, + "theoretical_loss": 3.301353900417967, + "tokens_seen": 3239575552 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019674935842600513, + "loss": 2.5602, + "theoretical_loss": 3.301343790290513, + "tokens_seen": 3239706624 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019632164242942688, + "loss": 2.6852, + "theoretical_loss": 3.3013336806866116, + "tokens_seen": 3239837696 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019589392643284858, + "loss": 2.6549, + "theoretical_loss": 3.301323571606213, + "tokens_seen": 3239968768 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019546621043627034, + "loss": 2.588, + "theoretical_loss": 3.3013134630492695, + "tokens_seen": 3240099840 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019503849443969204, + "loss": 2.6043, + "theoretical_loss": 3.301303355015733, + "tokens_seen": 3240230912 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001946107784431138, + "loss": 2.5788, + "theoretical_loss": 3.3012932475055554, + "tokens_seen": 3240361984 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001941830624465355, + "loss": 2.538, + "theoretical_loss": 3.301283140518688, + "tokens_seen": 3240493056 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019375534644995723, + "loss": 2.6047, + "theoretical_loss": 3.3012730340550824, + "tokens_seen": 3240624128 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019332763045337896, + "loss": 2.642, + "theoretical_loss": 3.3012629281146912, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019289991445680069, + "loss": 2.5193, + "theoretical_loss": 3.301252822697465, + "tokens_seen": 3240886272 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019247219846022241, + "loss": 2.5755, + "theoretical_loss": 3.3012427178033565, + "tokens_seen": 3241017344 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1773780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.509438991546631, + "objective/train/theoretical_loss": 3.301237665552456, + "objective/train/tokens_used": 271134176, + "theoretical_loss": 3.301237665552456, + "tokens_seen": 3241082880 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019204448246364414, + "loss": 2.5439, + "theoretical_loss": 3.3012326134323167, + "tokens_seen": 3241148416 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019161676646706587, + "loss": 2.5383, + "theoretical_loss": 3.301222509584298, + "tokens_seen": 3241279488 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001911890504704876, + "loss": 2.4258, + "theoretical_loss": 3.301212406259252, + "tokens_seen": 3241410560 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019076133447390933, + "loss": 2.5851, + "theoretical_loss": 3.3012023034571305, + "tokens_seen": 3241541632 + }, + { + "epoch": 0.81, + "learning_rate": 0.00019033361847733106, + "loss": 2.6308, + "theoretical_loss": 3.3011922011778854, + "tokens_seen": 3241672704 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001899059024807528, + "loss": 2.5062, + "theoretical_loss": 3.301182099421468, + "tokens_seen": 3241803776 + }, + { + "epoch": 0.81, + "learning_rate": 0.00018947818648417452, + "loss": 2.7067, + "theoretical_loss": 3.3011719981878302, + "tokens_seen": 3241934848 + }, + { + "epoch": 0.81, + "learning_rate": 0.00018905047048759624, + "loss": 2.5671, + "theoretical_loss": 3.3011618974769243, + "tokens_seen": 3242065920 + }, + { + "epoch": 0.81, + "learning_rate": 0.00018862275449101795, + "loss": 2.4739, + "theoretical_loss": 3.301151797288702, + "tokens_seen": 3242196992 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001881950384944397, + "loss": 2.7524, + "theoretical_loss": 3.301141697623115, + "tokens_seen": 3242328064 + }, + { + "epoch": 0.81, + "learning_rate": 0.00018776732249786143, + "loss": 2.4842, + "theoretical_loss": 3.301131598480115, + "tokens_seen": 3242459136 + }, + { + "epoch": 0.81, + "learning_rate": 0.00018733960650128316, + "loss": 2.5304, + "theoretical_loss": 3.301121499859654, + "tokens_seen": 3242590208 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1774468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.266578197479248, + "objective/train/theoretical_loss": 3.3011114017616836, + "objective/train/tokens_used": 272772576, + "theoretical_loss": 3.3011114017616836, + "tokens_seen": 3242721280 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001869118905047049, + "loss": 2.6048, + "theoretical_loss": 3.3011114017616836, + "tokens_seen": 3242721280 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001864841745081266, + "loss": 2.5415, + "theoretical_loss": 3.301101304186156, + "tokens_seen": 3242852352 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018605645851154835, + "loss": 2.5032, + "theoretical_loss": 3.3010912071330227, + "tokens_seen": 3242983424 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018562874251497005, + "loss": 2.4055, + "theoretical_loss": 3.301081110602236, + "tokens_seen": 3243114496 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001852010265183918, + "loss": 2.5573, + "theoretical_loss": 3.301071014593748, + "tokens_seen": 3243245568 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001847733105218135, + "loss": 2.4734, + "theoretical_loss": 3.3010609191075093, + "tokens_seen": 3243376640 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018434559452523526, + "loss": 2.4399, + "theoretical_loss": 3.3010508241434726, + "tokens_seen": 3243507712 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018391787852865696, + "loss": 2.4637, + "theoretical_loss": 3.3010407297015902, + "tokens_seen": 3243638784 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001834901625320787, + "loss": 2.58, + "theoretical_loss": 3.3010306357818138, + "tokens_seen": 3243769856 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018306244653550045, + "loss": 2.6065, + "theoretical_loss": 3.3010205423840944, + "tokens_seen": 3243900928 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018263473053892215, + "loss": 2.5627, + "theoretical_loss": 3.301010449508385, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001822070145423439, + "loss": 2.5044, + "theoretical_loss": 3.3010003571546367, + "tokens_seen": 3244163072 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001817792985457656, + "loss": 2.6545, + "theoretical_loss": 3.3009902653228025, + "tokens_seen": 3244294144 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1775745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9227540493011475, + "objective/train/theoretical_loss": 3.3009852196025875, + "objective/train/tokens_used": 274410976, + "theoretical_loss": 3.3009852196025875, + "tokens_seen": 3244359680 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018135158254918736, + "loss": 2.5147, + "theoretical_loss": 3.300980174012833, + "tokens_seen": 3244425216 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018092386655260906, + "loss": 2.4677, + "theoretical_loss": 3.300970083224681, + "tokens_seen": 3244556288 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001804961505560308, + "loss": 2.5792, + "theoretical_loss": 3.300959992958298, + "tokens_seen": 3244687360 + }, + { + "epoch": 0.82, + "learning_rate": 0.00018006843455945252, + "loss": 2.5536, + "theoretical_loss": 3.3009499032136365, + "tokens_seen": 3244818432 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017964071856287425, + "loss": 2.6178, + "theoretical_loss": 3.300939813990648, + "tokens_seen": 3244949504 + }, + { + "epoch": 0.82, + "learning_rate": 0.000179213002566296, + "loss": 2.4734, + "theoretical_loss": 3.300929725289284, + "tokens_seen": 3245080576 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001787852865697177, + "loss": 2.657, + "theoretical_loss": 3.300919637109498, + "tokens_seen": 3245211648 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017835757057313946, + "loss": 2.4026, + "theoretical_loss": 3.3009095494512404, + "tokens_seen": 3245342720 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017792985457656116, + "loss": 2.5456, + "theoretical_loss": 3.300899462314464, + "tokens_seen": 3245473792 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001775021385799829, + "loss": 2.6093, + "theoretical_loss": 3.3008893756991204, + "tokens_seen": 3245604864 + }, + { + "epoch": 0.82, + "learning_rate": 0.00017707442258340462, + "loss": 2.4946, + "theoretical_loss": 3.3008792896051617, + "tokens_seen": 3245735936 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017664670658682635, + "loss": 2.6761, + "theoretical_loss": 3.30086920403254, + "tokens_seen": 3245867008 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1776229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.64058518409729, + "objective/train/theoretical_loss": 3.300859118981207, + "objective/train/tokens_used": 276049376, + "theoretical_loss": 3.300859118981207, + "tokens_seen": 3245998080 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017621899059024808, + "loss": 2.3656, + "theoretical_loss": 3.300859118981207, + "tokens_seen": 3245998080 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001757912745936698, + "loss": 2.6333, + "theoretical_loss": 3.300849034451115, + "tokens_seen": 3246129152 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001753635585970915, + "loss": 2.5949, + "theoretical_loss": 3.3008389504422166, + "tokens_seen": 3246260224 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017493584260051327, + "loss": 2.5474, + "theoretical_loss": 3.300828866954463, + "tokens_seen": 3246391296 + }, + { + "epoch": 0.83, + "learning_rate": 0.000174508126603935, + "loss": 2.5367, + "theoretical_loss": 3.300818783987806, + "tokens_seen": 3246522368 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017408041060735672, + "loss": 2.5083, + "theoretical_loss": 3.300808701542198, + "tokens_seen": 3246653440 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017365269461077845, + "loss": 2.5172, + "theoretical_loss": 3.3007986196175914, + "tokens_seen": 3246784512 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017322497861420018, + "loss": 2.5857, + "theoretical_loss": 3.300788538213938, + "tokens_seen": 3246915584 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001727972626176219, + "loss": 2.5581, + "theoretical_loss": 3.3007784573311896, + "tokens_seen": 3247046656 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001723695466210436, + "loss": 2.5104, + "theoretical_loss": 3.3007683769692986, + "tokens_seen": 3247177728 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017194183062446537, + "loss": 2.5134, + "theoretical_loss": 3.3007582971282163, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017151411462788707, + "loss": 2.4078, + "theoretical_loss": 3.300748217807896, + "tokens_seen": 3247439872 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017108639863130882, + "loss": 2.4862, + "theoretical_loss": 3.3007381390082893, + "tokens_seen": 3247570944 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1777442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575650691986084, + "objective/train/theoretical_loss": 3.3007330998037383, + "objective/train/tokens_used": 277687776, + "theoretical_loss": 3.3007330998037383, + "tokens_seen": 3247636480 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017065868263473055, + "loss": 2.7088, + "theoretical_loss": 3.300728060729348, + "tokens_seen": 3247702016 + }, + { + "epoch": 0.83, + "learning_rate": 0.00017023096663815225, + "loss": 2.5033, + "theoretical_loss": 3.3007179829710243, + "tokens_seen": 3247833088 + }, + { + "epoch": 0.83, + "learning_rate": 0.000169803250641574, + "loss": 2.6317, + "theoretical_loss": 3.3007079057332698, + "tokens_seen": 3247964160 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001693755346449957, + "loss": 2.5483, + "theoretical_loss": 3.3006978290160376, + "tokens_seen": 3248095232 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016894781864841747, + "loss": 2.476, + "theoretical_loss": 3.3006877528192797, + "tokens_seen": 3248226304 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016852010265183917, + "loss": 2.4206, + "theoretical_loss": 3.3006776771429474, + "tokens_seen": 3248357376 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016809238665526093, + "loss": 2.5209, + "theoretical_loss": 3.3006676019869934, + "tokens_seen": 3248488448 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016766467065868263, + "loss": 2.573, + "theoretical_loss": 3.30065752735137, + "tokens_seen": 3248619520 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016723695466210436, + "loss": 2.5961, + "theoretical_loss": 3.3006474532360284, + "tokens_seen": 3248750592 + }, + { + "epoch": 0.83, + "learning_rate": 0.00016680923866552608, + "loss": 2.4741, + "theoretical_loss": 3.300637379640922, + "tokens_seen": 3248881664 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016638152266894781, + "loss": 2.5446, + "theoretical_loss": 3.300627306566002, + "tokens_seen": 3249012736 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016595380667236957, + "loss": 2.6699, + "theoretical_loss": 3.300617234011221, + "tokens_seen": 3249143808 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1777962, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4572715759277344, + "objective/train/theoretical_loss": 3.300607161976531, + "objective/train/tokens_used": 279326176, + "theoretical_loss": 3.300607161976531, + "tokens_seen": 3249274880 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016552609067579127, + "loss": 2.5116, + "theoretical_loss": 3.300607161976531, + "tokens_seen": 3249274880 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016509837467921303, + "loss": 2.6514, + "theoretical_loss": 3.3005970904618844, + "tokens_seen": 3249405952 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016467065868263473, + "loss": 2.556, + "theoretical_loss": 3.300587019467233, + "tokens_seen": 3249537024 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016424294268605646, + "loss": 2.5298, + "theoretical_loss": 3.3005769489925294, + "tokens_seen": 3249668096 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016381522668947819, + "loss": 2.4935, + "theoretical_loss": 3.300566879037725, + "tokens_seen": 3249799168 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016338751069289991, + "loss": 2.5608, + "theoretical_loss": 3.3005568096027735, + "tokens_seen": 3249930240 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016295979469632164, + "loss": 2.5232, + "theoretical_loss": 3.3005467406876257, + "tokens_seen": 3250061312 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016253207869974337, + "loss": 2.6231, + "theoretical_loss": 3.300536672292234, + "tokens_seen": 3250192384 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016210436270316513, + "loss": 2.6052, + "theoretical_loss": 3.300526604416551, + "tokens_seen": 3250323456 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016167664670658683, + "loss": 2.4848, + "theoretical_loss": 3.300516537060529, + "tokens_seen": 3250454528 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016124893071000856, + "loss": 2.5134, + "theoretical_loss": 3.3005064702241196, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001608212147134303, + "loss": 2.3293, + "theoretical_loss": 3.300496403907276, + "tokens_seen": 3250716672 + }, + { + "epoch": 0.84, + "learning_rate": 0.00016039349871685202, + "loss": 2.5666, + "theoretical_loss": 3.3004863381099496, + "tokens_seen": 3250847744 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1779321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.88250994682312, + "objective/train/theoretical_loss": 3.3004813054060906, + "objective/train/tokens_used": 280964576, + "theoretical_loss": 3.3004813054060906, + "tokens_seen": 3250913280 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015996578272027374, + "loss": 2.6959, + "theoretical_loss": 3.3004762728320927, + "tokens_seen": 3250978816 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015953806672369547, + "loss": 2.6402, + "theoretical_loss": 3.300466208073658, + "tokens_seen": 3251109888 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015911035072711718, + "loss": 2.6419, + "theoretical_loss": 3.300456143834598, + "tokens_seen": 3251240960 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015868263473053893, + "loss": 2.4547, + "theoretical_loss": 3.300446080114864, + "tokens_seen": 3251372032 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015825491873396063, + "loss": 2.4239, + "theoretical_loss": 3.300436016914409, + "tokens_seen": 3251503104 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001578272027373824, + "loss": 2.5422, + "theoretical_loss": 3.3004259542331846, + "tokens_seen": 3251634176 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015739948674080412, + "loss": 2.5797, + "theoretical_loss": 3.3004158920711437, + "tokens_seen": 3251765248 + }, + { + "epoch": 0.84, + "learning_rate": 0.00015697177074422585, + "loss": 2.5399, + "theoretical_loss": 3.3004058304282387, + "tokens_seen": 3251896320 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015654405474764757, + "loss": 2.6385, + "theoretical_loss": 3.300395769304421, + "tokens_seen": 3252027392 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015611633875106928, + "loss": 2.5484, + "theoretical_loss": 3.300385708699644, + "tokens_seen": 3252158464 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015568862275449103, + "loss": 2.4785, + "theoretical_loss": 3.3003756486138593, + "tokens_seen": 3252289536 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015526090675791273, + "loss": 2.3938, + "theoretical_loss": 3.3003655890470194, + "tokens_seen": 3252420608 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1779836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.387849807739258, + "objective/train/theoretical_loss": 3.3003555299990768, + "objective/train/tokens_used": 282602976, + "theoretical_loss": 3.3003555299990768, + "tokens_seen": 3252551680 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001548331907613345, + "loss": 2.4833, + "theoretical_loss": 3.3003555299990768, + "tokens_seen": 3252551680 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001544054747647562, + "loss": 2.5641, + "theoretical_loss": 3.3003454714699836, + "tokens_seen": 3252682752 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015397775876817792, + "loss": 2.5312, + "theoretical_loss": 3.300335413459692, + "tokens_seen": 3252813824 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015355004277159968, + "loss": 2.5279, + "theoretical_loss": 3.3003253559681545, + "tokens_seen": 3252944896 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015312232677502138, + "loss": 2.4937, + "theoretical_loss": 3.300315298995324, + "tokens_seen": 3253075968 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015269461077844313, + "loss": 2.3691, + "theoretical_loss": 3.300305242541152, + "tokens_seen": 3253207040 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015226689478186484, + "loss": 2.5269, + "theoretical_loss": 3.3002951866055907, + "tokens_seen": 3253338112 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001518391787852866, + "loss": 2.6473, + "theoretical_loss": 3.300285131188594, + "tokens_seen": 3253469184 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001514114627887083, + "loss": 2.4821, + "theoretical_loss": 3.300275076290112, + "tokens_seen": 3253600256 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015098374679213002, + "loss": 2.5156, + "theoretical_loss": 3.300265021910099, + "tokens_seen": 3253731328 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015055603079555175, + "loss": 2.5913, + "theoretical_loss": 3.300254968048507, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00015012831479897348, + "loss": 2.7976, + "theoretical_loss": 3.3002449147052872, + "tokens_seen": 3253993472 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001497005988023952, + "loss": 2.5524, + "theoretical_loss": 3.3002348618803934, + "tokens_seen": 3254124544 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1781080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.747638463973999, + "objective/train/theoretical_loss": 3.3002298356623037, + "objective/train/tokens_used": 284241376, + "theoretical_loss": 3.3002298356623037, + "tokens_seen": 3254190080 + }, + { + "epoch": 0.85, + "learning_rate": 0.00014927288280581694, + "loss": 2.4021, + "theoretical_loss": 3.300224809573777, + "tokens_seen": 3254255616 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001488451668092387, + "loss": 2.6011, + "theoretical_loss": 3.3002147577853913, + "tokens_seen": 3254386688 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001484174508126604, + "loss": 2.5126, + "theoretical_loss": 3.300204706515188, + "tokens_seen": 3254517760 + }, + { + "epoch": 0.85, + "learning_rate": 0.00014798973481608212, + "loss": 2.5876, + "theoretical_loss": 3.30019465576312, + "tokens_seen": 3254648832 + }, + { + "epoch": 0.85, + "learning_rate": 0.00014756201881950385, + "loss": 2.6212, + "theoretical_loss": 3.3001846055291395, + "tokens_seen": 3254779904 + }, + { + "epoch": 0.85, + "learning_rate": 0.00014713430282292558, + "loss": 2.6038, + "theoretical_loss": 3.300174555813199, + "tokens_seen": 3254910976 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001467065868263473, + "loss": 2.5393, + "theoretical_loss": 3.3001645066152507, + "tokens_seen": 3255042048 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014627887082976904, + "loss": 2.4902, + "theoretical_loss": 3.300154457935247, + "tokens_seen": 3255173120 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014585115483319074, + "loss": 2.5368, + "theoretical_loss": 3.300144409773141, + "tokens_seen": 3255304192 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001454234388366125, + "loss": 2.5369, + "theoretical_loss": 3.300134362128885, + "tokens_seen": 3255435264 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014499572284003422, + "loss": 2.3826, + "theoretical_loss": 3.3001243150024306, + "tokens_seen": 3255566336 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014456800684345595, + "loss": 2.4719, + "theoretical_loss": 3.300114268393731, + "tokens_seen": 3255697408 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1781747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3552169799804688, + "objective/train/theoretical_loss": 3.3001042223027386, + "objective/train/tokens_used": 285879776, + "theoretical_loss": 3.3001042223027386, + "tokens_seen": 3255828480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014414029084687768, + "loss": 2.4574, + "theoretical_loss": 3.3001042223027386, + "tokens_seen": 3255828480 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001437125748502994, + "loss": 2.5886, + "theoretical_loss": 3.300094176729406, + "tokens_seen": 3255959552 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014328485885372114, + "loss": 2.4677, + "theoretical_loss": 3.3000841316736853, + "tokens_seen": 3256090624 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014285714285714284, + "loss": 2.4431, + "theoretical_loss": 3.3000740871355294, + "tokens_seen": 3256221696 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001424294268605646, + "loss": 2.5458, + "theoretical_loss": 3.30006404311489, + "tokens_seen": 3256352768 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001420017108639863, + "loss": 2.5976, + "theoretical_loss": 3.300053999611721, + "tokens_seen": 3256483840 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014157399486740805, + "loss": 2.4439, + "theoretical_loss": 3.3000439566259736, + "tokens_seen": 3256614912 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014114627887082976, + "loss": 2.5097, + "theoretical_loss": 3.300033914157601, + "tokens_seen": 3256745984 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001407185628742515, + "loss": 2.4835, + "theoretical_loss": 3.3000238722065562, + "tokens_seen": 3256877056 + }, + { + "epoch": 0.86, + "learning_rate": 0.00014029084687767324, + "loss": 2.4434, + "theoretical_loss": 3.3000138307727904, + "tokens_seen": 3257008128 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013986313088109494, + "loss": 2.628, + "theoretical_loss": 3.300003789856257, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001394354148845167, + "loss": 2.6804, + "theoretical_loss": 3.2999937494569087, + "tokens_seen": 3257270272 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001390076988879384, + "loss": 2.6107, + "theoretical_loss": 3.2999837095746973, + "tokens_seen": 3257401344 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1782888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.327956438064575, + "objective/train/theoretical_loss": 3.2999786898275034, + "objective/train/tokens_used": 287518176, + "theoretical_loss": 3.2999786898275034, + "tokens_seen": 3257466880 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013857998289136016, + "loss": 2.5895, + "theoretical_loss": 3.299973670209576, + "tokens_seen": 3257532416 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013815226689478186, + "loss": 2.5606, + "theoretical_loss": 3.2999636313614973, + "tokens_seen": 3257663488 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013772455089820359, + "loss": 2.55, + "theoretical_loss": 3.2999535930304136, + "tokens_seen": 3257794560 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013729683490162531, + "loss": 2.5115, + "theoretical_loss": 3.2999435552162772, + "tokens_seen": 3257925632 + }, + { + "epoch": 0.86, + "learning_rate": 0.00013686911890504704, + "loss": 2.457, + "theoretical_loss": 3.299933517919041, + "tokens_seen": 3258056704 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001364414029084688, + "loss": 2.6031, + "theoretical_loss": 3.299923481138658, + "tokens_seen": 3258187776 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001360136869118905, + "loss": 2.536, + "theoretical_loss": 3.29991344487508, + "tokens_seen": 3258318848 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013558597091531226, + "loss": 2.5729, + "theoretical_loss": 3.29990340912826, + "tokens_seen": 3258449920 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013515825491873396, + "loss": 2.4097, + "theoretical_loss": 3.299893373898151, + "tokens_seen": 3258580992 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001347305389221557, + "loss": 2.7433, + "theoretical_loss": 3.299883339184705, + "tokens_seen": 3258712064 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013430282292557742, + "loss": 2.5549, + "theoretical_loss": 3.299873304987875, + "tokens_seen": 3258843136 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013387510692899914, + "loss": 2.4077, + "theoretical_loss": 3.299863271307613, + "tokens_seen": 3258974208 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1784219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6288301944732666, + "objective/train/theoretical_loss": 3.2998532381438723, + "objective/train/tokens_used": 289156576, + "theoretical_loss": 3.2998532381438723, + "tokens_seen": 3259105280 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013344739093242087, + "loss": 2.6571, + "theoretical_loss": 3.2998532381438723, + "tokens_seen": 3259105280 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001330196749358426, + "loss": 2.6846, + "theoretical_loss": 3.299843205496605, + "tokens_seen": 3259236352 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013259195893926433, + "loss": 2.4786, + "theoretical_loss": 3.299833173365765, + "tokens_seen": 3259367424 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013216424294268606, + "loss": 2.4946, + "theoretical_loss": 3.299823141751303, + "tokens_seen": 3259498496 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001317365269461078, + "loss": 2.5974, + "theoretical_loss": 3.299813110653173, + "tokens_seen": 3259629568 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013130881094952952, + "loss": 2.5487, + "theoretical_loss": 3.2998030800713276, + "tokens_seen": 3259760640 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013088109495295125, + "loss": 2.5281, + "theoretical_loss": 3.299793050005719, + "tokens_seen": 3259891712 + }, + { + "epoch": 0.87, + "learning_rate": 0.00013045337895637297, + "loss": 2.469, + "theoretical_loss": 3.2997830204563, + "tokens_seen": 3260022784 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001300256629597947, + "loss": 2.607, + "theoretical_loss": 3.2997729914230236, + "tokens_seen": 3260153856 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001295979469632164, + "loss": 2.5748, + "theoretical_loss": 3.299762962905842, + "tokens_seen": 3260284928 + }, + { + "epoch": 0.87, + "learning_rate": 0.00012917023096663816, + "loss": 2.5545, + "theoretical_loss": 3.299752934904708, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00012874251497005986, + "loss": 2.5687, + "theoretical_loss": 3.299742907419575, + "tokens_seen": 3260547072 + }, + { + "epoch": 0.87, + "learning_rate": 0.00012831479897348162, + "loss": 2.6328, + "theoretical_loss": 3.299732880450395, + "tokens_seen": 3260678144 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1784817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5144858360290527, + "objective/train/theoretical_loss": 3.2997278671592727, + "objective/train/tokens_used": 290794976, + "theoretical_loss": 3.2997278671592727, + "tokens_seen": 3260743680 + }, + { + "epoch": 0.87, + "learning_rate": 0.00012788708297690335, + "loss": 2.4409, + "theoretical_loss": 3.2997228539971206, + "tokens_seen": 3260809216 + }, + { + "epoch": 0.87, + "learning_rate": 0.00012745936698032508, + "loss": 2.6368, + "theoretical_loss": 3.299712828059705, + "tokens_seen": 3260940288 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001270316509837468, + "loss": 2.5522, + "theoretical_loss": 3.299702802638101, + "tokens_seen": 3261071360 + }, + { + "epoch": 0.87, + "learning_rate": 0.0001266039349871685, + "loss": 2.462, + "theoretical_loss": 3.29969277773226, + "tokens_seen": 3261202432 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012617621899059026, + "loss": 2.4761, + "theoretical_loss": 3.299682753342137, + "tokens_seen": 3261333504 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012574850299401196, + "loss": 2.6031, + "theoretical_loss": 3.299672729467683, + "tokens_seen": 3261464576 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012532078699743372, + "loss": 2.4891, + "theoretical_loss": 3.299662706108851, + "tokens_seen": 3261595648 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012489307100085545, + "loss": 2.5886, + "theoretical_loss": 3.2996526832655944, + "tokens_seen": 3261726720 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012446535500427718, + "loss": 2.5875, + "theoretical_loss": 3.2996426609378657, + "tokens_seen": 3261857792 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012403763900769888, + "loss": 2.5543, + "theoretical_loss": 3.299632639125617, + "tokens_seen": 3261988864 + }, + { + "epoch": 0.88, + "learning_rate": 0.0001236099230111206, + "loss": 2.4939, + "theoretical_loss": 3.2996226178288026, + "tokens_seen": 3262119936 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012318220701454234, + "loss": 2.5754, + "theoretical_loss": 3.299612597047374, + "tokens_seen": 3262251008 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1786311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3557615280151367, + "objective/train/theoretical_loss": 3.2996025767812838, + "objective/train/tokens_used": 292433376, + "theoretical_loss": 3.2996025767812838, + "tokens_seen": 3262382080 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012275449101796406, + "loss": 2.5092, + "theoretical_loss": 3.2996025767812838, + "tokens_seen": 3262382080 + }, + { + "epoch": 0.88, + "learning_rate": 0.0001223267750213858, + "loss": 2.7033, + "theoretical_loss": 3.299592557030486, + "tokens_seen": 3262513152 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012189905902480752, + "loss": 2.3802, + "theoretical_loss": 3.299582537794932, + "tokens_seen": 3262644224 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012147134302822926, + "loss": 2.6566, + "theoretical_loss": 3.2995725190745757, + "tokens_seen": 3262775296 + }, + { + "epoch": 0.88, + "learning_rate": 0.000121043627031651, + "loss": 2.2799, + "theoretical_loss": 3.29956250086937, + "tokens_seen": 3262906368 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012061591103507272, + "loss": 2.4147, + "theoretical_loss": 3.2995524831792666, + "tokens_seen": 3263037440 + }, + { + "epoch": 0.88, + "learning_rate": 0.00012018819503849444, + "loss": 2.5069, + "theoretical_loss": 3.299542466004219, + "tokens_seen": 3263168512 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011976047904191617, + "loss": 2.3787, + "theoretical_loss": 3.29953244934418, + "tokens_seen": 3263299584 + }, + { + "epoch": 0.88, + "learning_rate": 0.0001193327630453379, + "loss": 2.4275, + "theoretical_loss": 3.299522433199103, + "tokens_seen": 3263430656 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011890504704875962, + "loss": 2.5042, + "theoretical_loss": 3.29951241756894, + "tokens_seen": 3263561728 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011847733105218135, + "loss": 2.4289, + "theoretical_loss": 3.299502402453644, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011804961505560308, + "loss": 2.5625, + "theoretical_loss": 3.299492387853168, + "tokens_seen": 3263823872 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011762189905902481, + "loss": 2.5523, + "theoretical_loss": 3.2994823737674652, + "tokens_seen": 3263954944 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1786881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3557543754577637, + "objective/train/theoretical_loss": 3.2994773669176385, + "objective/train/tokens_used": 294071776, + "theoretical_loss": 3.2994773669176385, + "tokens_seen": 3264020480 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011719418306244654, + "loss": 2.4299, + "theoretical_loss": 3.2994723601964875, + "tokens_seen": 3264086016 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011676646706586827, + "loss": 2.4991, + "theoretical_loss": 3.2994623471401887, + "tokens_seen": 3264217088 + }, + { + "epoch": 0.88, + "learning_rate": 0.00011633875106929, + "loss": 2.6179, + "theoretical_loss": 3.299452334598522, + "tokens_seen": 3264348160 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011591103507271173, + "loss": 2.4253, + "theoretical_loss": 3.299442322571439, + "tokens_seen": 3264479232 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011548331907613345, + "loss": 2.4555, + "theoretical_loss": 3.2994323110588932, + "tokens_seen": 3264610304 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011505560307955518, + "loss": 2.6147, + "theoretical_loss": 3.2994223000608383, + "tokens_seen": 3264741376 + }, + { + "epoch": 0.89, + "learning_rate": 0.0001146278870829769, + "loss": 2.4139, + "theoretical_loss": 3.2994122895772255, + "tokens_seen": 3264872448 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011420017108639863, + "loss": 2.4572, + "theoretical_loss": 3.2994022796080094, + "tokens_seen": 3265003520 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011377245508982036, + "loss": 2.5296, + "theoretical_loss": 3.2993922701531417, + "tokens_seen": 3265134592 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011334473909324208, + "loss": 2.5373, + "theoretical_loss": 3.2993822612125765, + "tokens_seen": 3265265664 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011291702309666383, + "loss": 2.4843, + "theoretical_loss": 3.2993722527862657, + "tokens_seen": 3265396736 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011248930710008556, + "loss": 2.5279, + "theoretical_loss": 3.2993622448741626, + "tokens_seen": 3265527808 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1787929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9586187601089478, + "objective/train/theoretical_loss": 3.2993522374762203, + "objective/train/tokens_used": 295710176, + "theoretical_loss": 3.2993522374762203, + "tokens_seen": 3265658880 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011206159110350727, + "loss": 2.3587, + "theoretical_loss": 3.2993522374762203, + "tokens_seen": 3265658880 + }, + { + "epoch": 0.89, + "learning_rate": 0.000111633875106929, + "loss": 2.5182, + "theoretical_loss": 3.2993422305923916, + "tokens_seen": 3265789952 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011120615911035073, + "loss": 2.4224, + "theoretical_loss": 3.299332224222629, + "tokens_seen": 3265921024 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011077844311377246, + "loss": 2.5394, + "theoretical_loss": 3.2993222183668864, + "tokens_seen": 3266052096 + }, + { + "epoch": 0.89, + "learning_rate": 0.00011035072711719419, + "loss": 2.5439, + "theoretical_loss": 3.2993122130251162, + "tokens_seen": 3266183168 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010992301112061591, + "loss": 2.6219, + "theoretical_loss": 3.2993022081972714, + "tokens_seen": 3266314240 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010949529512403764, + "loss": 2.5979, + "theoretical_loss": 3.299292203883305, + "tokens_seen": 3266445312 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010906757912745936, + "loss": 2.5511, + "theoretical_loss": 3.2992822000831703, + "tokens_seen": 3266576384 + }, + { + "epoch": 0.89, + "learning_rate": 0.0001086398631308811, + "loss": 2.531, + "theoretical_loss": 3.2992721967968195, + "tokens_seen": 3266707456 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010821214713430283, + "loss": 2.6969, + "theoretical_loss": 3.2992621940242066, + "tokens_seen": 3266838528 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010778443113772456, + "loss": 2.4865, + "theoretical_loss": 3.2992521917652837, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010735671514114629, + "loss": 2.4688, + "theoretical_loss": 3.2992421900200046, + "tokens_seen": 3267100672 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010692899914456802, + "loss": 2.6101, + "theoretical_loss": 3.299232188788322, + "tokens_seen": 3267231744 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1788521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7996864318847656, + "objective/train/theoretical_loss": 3.299227188365064, + "objective/train/tokens_used": 297348576, + "theoretical_loss": 3.299227188365064, + "tokens_seen": 3267297280 + }, + { + "epoch": 0.89, + "learning_rate": 0.00010650128314798973, + "loss": 2.4955, + "theoretical_loss": 3.2992221880701886, + "tokens_seen": 3267362816 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010607356715141146, + "loss": 2.5531, + "theoretical_loss": 3.2992121878655576, + "tokens_seen": 3267493888 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010564585115483319, + "loss": 2.4553, + "theoretical_loss": 3.2992021881743825, + "tokens_seen": 3267624960 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010521813515825492, + "loss": 2.4558, + "theoretical_loss": 3.299192188996616, + "tokens_seen": 3267756032 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010479041916167665, + "loss": 2.5602, + "theoretical_loss": 3.299182190332211, + "tokens_seen": 3267887104 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010436270316509839, + "loss": 2.5385, + "theoretical_loss": 3.2991721921811203, + "tokens_seen": 3268018176 + }, + { + "epoch": 0.9, + "learning_rate": 0.0001039349871685201, + "loss": 2.4475, + "theoretical_loss": 3.299162194543298, + "tokens_seen": 3268149248 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010350727117194183, + "loss": 2.4763, + "theoretical_loss": 3.299152197418696, + "tokens_seen": 3268280320 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010307955517536356, + "loss": 2.5838, + "theoretical_loss": 3.2991422008072684, + "tokens_seen": 3268411392 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010265183917878529, + "loss": 2.5198, + "theoretical_loss": 3.2991322047089673, + "tokens_seen": 3268542464 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010222412318220702, + "loss": 2.4978, + "theoretical_loss": 3.2991222091237464, + "tokens_seen": 3268673536 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010179640718562875, + "loss": 2.573, + "theoretical_loss": 3.2991122140515587, + "tokens_seen": 3268804608 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1789746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.044440507888794, + "objective/train/theoretical_loss": 3.2991022194923576, + "objective/train/tokens_used": 298986976, + "theoretical_loss": 3.2991022194923576, + "tokens_seen": 3268935680 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010136869118905046, + "loss": 2.5106, + "theoretical_loss": 3.2991022194923576, + "tokens_seen": 3268935680 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010094097519247219, + "loss": 2.515, + "theoretical_loss": 3.2990922254460955, + "tokens_seen": 3269066752 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010051325919589392, + "loss": 2.5237, + "theoretical_loss": 3.2990822319127258, + "tokens_seen": 3269197824 + }, + { + "epoch": 0.9, + "learning_rate": 0.00010008554319931566, + "loss": 2.3423, + "theoretical_loss": 3.299072238892202, + "tokens_seen": 3269328896 + }, + { + "epoch": 0.9, + "learning_rate": 9.965782720273739e-05, + "loss": 2.5378, + "theoretical_loss": 3.2990622463844765, + "tokens_seen": 3269459968 + }, + { + "epoch": 0.9, + "learning_rate": 9.923011120615912e-05, + "loss": 2.5356, + "theoretical_loss": 3.2990522543895033, + "tokens_seen": 3269591040 + }, + { + "epoch": 0.9, + "learning_rate": 9.880239520958085e-05, + "loss": 2.51, + "theoretical_loss": 3.2990422629072347, + "tokens_seen": 3269722112 + }, + { + "epoch": 0.9, + "learning_rate": 9.837467921300256e-05, + "loss": 2.4805, + "theoretical_loss": 3.2990322719376244, + "tokens_seen": 3269853184 + }, + { + "epoch": 0.9, + "learning_rate": 9.794696321642429e-05, + "loss": 2.409, + "theoretical_loss": 3.2990222814806254, + "tokens_seen": 3269984256 + }, + { + "epoch": 0.9, + "learning_rate": 9.751924721984602e-05, + "loss": 2.4939, + "theoretical_loss": 3.2990122915361906, + "tokens_seen": 3270115328 + }, + { + "epoch": 0.9, + "learning_rate": 9.709153122326775e-05, + "loss": 2.5024, + "theoretical_loss": 3.2990023021042734, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.9, + "learning_rate": 9.666381522668948e-05, + "loss": 2.501, + "theoretical_loss": 3.298992313184827, + "tokens_seen": 3270377472 + }, + { + "epoch": 0.9, + "learning_rate": 9.623609923011121e-05, + "loss": 2.6065, + "theoretical_loss": 3.2989823247778043, + "tokens_seen": 3270508544 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1790362, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.279358386993408, + "objective/train/theoretical_loss": 3.298977330766437, + "objective/train/tokens_used": 300625376, + "theoretical_loss": 3.298977330766437, + "tokens_seen": 3270574080 + }, + { + "epoch": 0.91, + "learning_rate": 9.580838323353294e-05, + "loss": 2.5237, + "theoretical_loss": 3.2989723368831587, + "tokens_seen": 3270639616 + }, + { + "epoch": 0.91, + "learning_rate": 9.538066723695466e-05, + "loss": 2.4181, + "theoretical_loss": 3.2989623495008438, + "tokens_seen": 3270770688 + }, + { + "epoch": 0.91, + "learning_rate": 9.49529512403764e-05, + "loss": 2.589, + "theoretical_loss": 3.2989523626308124, + "tokens_seen": 3270901760 + }, + { + "epoch": 0.91, + "learning_rate": 9.452523524379812e-05, + "loss": 2.4541, + "theoretical_loss": 3.298942376273017, + "tokens_seen": 3271032832 + }, + { + "epoch": 0.91, + "learning_rate": 9.409751924721985e-05, + "loss": 2.6645, + "theoretical_loss": 3.298932390427412, + "tokens_seen": 3271163904 + }, + { + "epoch": 0.91, + "learning_rate": 9.366980325064158e-05, + "loss": 2.5151, + "theoretical_loss": 3.2989224050939496, + "tokens_seen": 3271294976 + }, + { + "epoch": 0.91, + "learning_rate": 9.32420872540633e-05, + "loss": 2.5406, + "theoretical_loss": 3.2989124202725844, + "tokens_seen": 3271426048 + }, + { + "epoch": 0.91, + "learning_rate": 9.281437125748502e-05, + "loss": 2.3461, + "theoretical_loss": 3.2989024359632677, + "tokens_seen": 3271557120 + }, + { + "epoch": 0.91, + "learning_rate": 9.238665526090675e-05, + "loss": 2.6028, + "theoretical_loss": 3.298892452165954, + "tokens_seen": 3271688192 + }, + { + "epoch": 0.91, + "learning_rate": 9.195893926432848e-05, + "loss": 2.6532, + "theoretical_loss": 3.2988824688805964, + "tokens_seen": 3271819264 + }, + { + "epoch": 0.91, + "learning_rate": 9.153122326775022e-05, + "loss": 2.4964, + "theoretical_loss": 3.298872486107148, + "tokens_seen": 3271950336 + }, + { + "epoch": 0.91, + "learning_rate": 9.110350727117195e-05, + "loss": 2.5105, + "theoretical_loss": 3.2988625038455623, + "tokens_seen": 3272081408 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1791336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585782051086426, + "objective/train/theoretical_loss": 3.298852522095792, + "objective/train/tokens_used": 302263776, + "theoretical_loss": 3.298852522095792, + "tokens_seen": 3272212480 + }, + { + "epoch": 0.91, + "learning_rate": 9.067579127459368e-05, + "loss": 2.4434, + "theoretical_loss": 3.298852522095792, + "tokens_seen": 3272212480 + }, + { + "epoch": 0.91, + "learning_rate": 9.02480752780154e-05, + "loss": 2.5535, + "theoretical_loss": 3.2988425408577906, + "tokens_seen": 3272343552 + }, + { + "epoch": 0.91, + "learning_rate": 8.982035928143712e-05, + "loss": 2.5515, + "theoretical_loss": 3.298832560131512, + "tokens_seen": 3272474624 + }, + { + "epoch": 0.91, + "learning_rate": 8.939264328485885e-05, + "loss": 2.3263, + "theoretical_loss": 3.298822579916908, + "tokens_seen": 3272605696 + }, + { + "epoch": 0.91, + "learning_rate": 8.896492728828058e-05, + "loss": 2.582, + "theoretical_loss": 3.2988126002139335, + "tokens_seen": 3272736768 + }, + { + "epoch": 0.91, + "learning_rate": 8.853721129170231e-05, + "loss": 2.449, + "theoretical_loss": 3.2988026210225407, + "tokens_seen": 3272867840 + }, + { + "epoch": 0.91, + "learning_rate": 8.810949529512404e-05, + "loss": 2.5481, + "theoretical_loss": 3.298792642342683, + "tokens_seen": 3272998912 + }, + { + "epoch": 0.91, + "learning_rate": 8.768177929854575e-05, + "loss": 2.5259, + "theoretical_loss": 3.2987826641743148, + "tokens_seen": 3273129984 + }, + { + "epoch": 0.91, + "learning_rate": 8.72540633019675e-05, + "loss": 2.445, + "theoretical_loss": 3.298772686517388, + "tokens_seen": 3273261056 + }, + { + "epoch": 0.91, + "learning_rate": 8.682634730538923e-05, + "loss": 2.7235, + "theoretical_loss": 3.2987627093718563, + "tokens_seen": 3273392128 + }, + { + "epoch": 0.91, + "learning_rate": 8.639863130881095e-05, + "loss": 2.6295, + "theoretical_loss": 3.298752732737673, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.91, + "learning_rate": 8.597091531223268e-05, + "loss": 2.4566, + "theoretical_loss": 3.298742756614792, + "tokens_seen": 3273654272 + }, + { + "epoch": 0.92, + "learning_rate": 8.554319931565441e-05, + "loss": 2.4428, + "theoretical_loss": 3.2987327810031664, + "tokens_seen": 3273785344 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1792685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3218390941619873, + "objective/train/theoretical_loss": 3.2987277933890593, + "objective/train/tokens_used": 303902176, + "theoretical_loss": 3.2987277933890593, + "tokens_seen": 3273850880 + }, + { + "epoch": 0.92, + "learning_rate": 8.511548331907613e-05, + "loss": 2.6666, + "theoretical_loss": 3.298722805902749, + "tokens_seen": 3273916416 + }, + { + "epoch": 0.92, + "learning_rate": 8.468776732249786e-05, + "loss": 2.5515, + "theoretical_loss": 3.2987128313134937, + "tokens_seen": 3274047488 + }, + { + "epoch": 0.92, + "learning_rate": 8.426005132591958e-05, + "loss": 2.5291, + "theoretical_loss": 3.2987028572353534, + "tokens_seen": 3274178560 + }, + { + "epoch": 0.92, + "learning_rate": 8.383233532934131e-05, + "loss": 2.6114, + "theoretical_loss": 3.2986928836682816, + "tokens_seen": 3274309632 + }, + { + "epoch": 0.92, + "learning_rate": 8.340461933276304e-05, + "loss": 2.6271, + "theoretical_loss": 3.2986829106122317, + "tokens_seen": 3274440704 + }, + { + "epoch": 0.92, + "learning_rate": 8.297690333618478e-05, + "loss": 2.5603, + "theoretical_loss": 3.2986729380671576, + "tokens_seen": 3274571776 + }, + { + "epoch": 0.92, + "learning_rate": 8.254918733960651e-05, + "loss": 2.5173, + "theoretical_loss": 3.2986629660330116, + "tokens_seen": 3274702848 + }, + { + "epoch": 0.92, + "learning_rate": 8.212147134302823e-05, + "loss": 2.5004, + "theoretical_loss": 3.298652994509748, + "tokens_seen": 3274833920 + }, + { + "epoch": 0.92, + "learning_rate": 8.169375534644996e-05, + "loss": 2.3853, + "theoretical_loss": 3.2986430234973194, + "tokens_seen": 3274964992 + }, + { + "epoch": 0.92, + "learning_rate": 8.126603934987169e-05, + "loss": 2.6222, + "theoretical_loss": 3.29863305299568, + "tokens_seen": 3275096064 + }, + { + "epoch": 0.92, + "learning_rate": 8.083832335329341e-05, + "loss": 2.5764, + "theoretical_loss": 3.2986230830047827, + "tokens_seen": 3275227136 + }, + { + "epoch": 0.92, + "learning_rate": 8.041060735671514e-05, + "loss": 2.4982, + "theoretical_loss": 3.298613113524581, + "tokens_seen": 3275358208 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1793234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.297673463821411, + "objective/train/theoretical_loss": 3.2986031445550283, + "objective/train/tokens_used": 305540576, + "theoretical_loss": 3.2986031445550283, + "tokens_seen": 3275489280 + }, + { + "epoch": 0.92, + "learning_rate": 7.998289136013687e-05, + "loss": 2.4303, + "theoretical_loss": 3.2986031445550283, + "tokens_seen": 3275489280 + }, + { + "epoch": 0.92, + "learning_rate": 7.955517536355859e-05, + "loss": 2.5015, + "theoretical_loss": 3.2985931760960776, + "tokens_seen": 3275620352 + }, + { + "epoch": 0.92, + "learning_rate": 7.912745936698032e-05, + "loss": 2.4736, + "theoretical_loss": 3.2985832081476834, + "tokens_seen": 3275751424 + }, + { + "epoch": 0.92, + "learning_rate": 7.869974337040206e-05, + "loss": 2.5, + "theoretical_loss": 3.298573240709798, + "tokens_seen": 3275882496 + }, + { + "epoch": 0.92, + "learning_rate": 7.827202737382379e-05, + "loss": 2.546, + "theoretical_loss": 3.2985632737823756, + "tokens_seen": 3276013568 + }, + { + "epoch": 0.92, + "learning_rate": 7.784431137724552e-05, + "loss": 2.5718, + "theoretical_loss": 3.2985533073653692, + "tokens_seen": 3276144640 + }, + { + "epoch": 0.92, + "learning_rate": 7.741659538066724e-05, + "loss": 2.5189, + "theoretical_loss": 3.298543341458732, + "tokens_seen": 3276275712 + }, + { + "epoch": 0.92, + "learning_rate": 7.698887938408896e-05, + "loss": 2.6189, + "theoretical_loss": 3.2985333760624185, + "tokens_seen": 3276406784 + }, + { + "epoch": 0.92, + "learning_rate": 7.656116338751069e-05, + "loss": 2.4942, + "theoretical_loss": 3.298523411176381, + "tokens_seen": 3276537856 + }, + { + "epoch": 0.92, + "learning_rate": 7.613344739093242e-05, + "loss": 2.5353, + "theoretical_loss": 3.2985134468005732, + "tokens_seen": 3276668928 + }, + { + "epoch": 0.93, + "learning_rate": 7.570573139435415e-05, + "loss": 2.5401, + "theoretical_loss": 3.298503482934949, + "tokens_seen": 3276800000 + }, + { + "epoch": 0.93, + "learning_rate": 7.527801539777588e-05, + "loss": 2.3956, + "theoretical_loss": 3.298493519579462, + "tokens_seen": 3276931072 + }, + { + "epoch": 0.93, + "learning_rate": 7.48502994011976e-05, + "loss": 2.4232, + "theoretical_loss": 3.298483556734065, + "tokens_seen": 3277062144 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1794479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3217952251434326, + "objective/train/theoretical_loss": 3.298478575502636, + "objective/train/tokens_used": 307178976, + "theoretical_loss": 3.298478575502636, + "tokens_seen": 3277127680 + }, + { + "epoch": 0.93, + "learning_rate": 7.442258340461935e-05, + "loss": 2.5046, + "theoretical_loss": 3.298473594398712, + "tokens_seen": 3277193216 + }, + { + "epoch": 0.93, + "learning_rate": 7.399486740804106e-05, + "loss": 2.4558, + "theoretical_loss": 3.2984636325733563, + "tokens_seen": 3277324288 + }, + { + "epoch": 0.93, + "learning_rate": 7.356715141146279e-05, + "loss": 2.5769, + "theoretical_loss": 3.2984536712579513, + "tokens_seen": 3277455360 + }, + { + "epoch": 0.93, + "learning_rate": 7.313943541488452e-05, + "loss": 2.3685, + "theoretical_loss": 3.2984437104524504, + "tokens_seen": 3277586432 + }, + { + "epoch": 0.93, + "learning_rate": 7.271171941830625e-05, + "loss": 2.525, + "theoretical_loss": 3.298433750156808, + "tokens_seen": 3277717504 + }, + { + "epoch": 0.93, + "learning_rate": 7.228400342172798e-05, + "loss": 2.4461, + "theoretical_loss": 3.2984237903709763, + "tokens_seen": 3277848576 + }, + { + "epoch": 0.93, + "learning_rate": 7.18562874251497e-05, + "loss": 2.378, + "theoretical_loss": 3.2984138310949094, + "tokens_seen": 3277979648 + }, + { + "epoch": 0.93, + "learning_rate": 7.142857142857142e-05, + "loss": 2.3551, + "theoretical_loss": 3.2984038723285614, + "tokens_seen": 3278110720 + }, + { + "epoch": 0.93, + "learning_rate": 7.100085543199315e-05, + "loss": 2.3994, + "theoretical_loss": 3.298393914071885, + "tokens_seen": 3278241792 + }, + { + "epoch": 0.93, + "learning_rate": 7.057313943541488e-05, + "loss": 2.3767, + "theoretical_loss": 3.2983839563248343, + "tokens_seen": 3278372864 + }, + { + "epoch": 0.93, + "learning_rate": 7.014542343883662e-05, + "loss": 2.5847, + "theoretical_loss": 3.2983739990873624, + "tokens_seen": 3278503936 + }, + { + "epoch": 0.93, + "learning_rate": 6.971770744225835e-05, + "loss": 2.6754, + "theoretical_loss": 3.298364042359423, + "tokens_seen": 3278635008 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1795059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5040347576141357, + "objective/train/theoretical_loss": 3.2983540861409697, + "objective/train/tokens_used": 308817376, + "theoretical_loss": 3.2983540861409697, + "tokens_seen": 3278766080 + }, + { + "epoch": 0.93, + "learning_rate": 6.928999144568008e-05, + "loss": 2.5056, + "theoretical_loss": 3.2983540861409697, + "tokens_seen": 3278766080 + }, + { + "epoch": 0.93, + "learning_rate": 6.886227544910179e-05, + "loss": 2.5205, + "theoretical_loss": 3.298344130431956, + "tokens_seen": 3278897152 + }, + { + "epoch": 0.93, + "learning_rate": 6.843455945252352e-05, + "loss": 2.545, + "theoretical_loss": 3.298334175232336, + "tokens_seen": 3279028224 + }, + { + "epoch": 0.93, + "learning_rate": 6.800684345594525e-05, + "loss": 2.6366, + "theoretical_loss": 3.298324220542063, + "tokens_seen": 3279159296 + }, + { + "epoch": 0.93, + "learning_rate": 6.757912745936698e-05, + "loss": 2.3622, + "theoretical_loss": 3.2983142663610896, + "tokens_seen": 3279290368 + }, + { + "epoch": 0.93, + "learning_rate": 6.715141146278871e-05, + "loss": 2.5523, + "theoretical_loss": 3.2983043126893703, + "tokens_seen": 3279421440 + }, + { + "epoch": 0.93, + "learning_rate": 6.672369546621044e-05, + "loss": 2.5336, + "theoretical_loss": 3.2982943595268592, + "tokens_seen": 3279552512 + }, + { + "epoch": 0.93, + "learning_rate": 6.629597946963217e-05, + "loss": 2.5485, + "theoretical_loss": 3.298284406873509, + "tokens_seen": 3279683584 + }, + { + "epoch": 0.93, + "learning_rate": 6.58682634730539e-05, + "loss": 2.591, + "theoretical_loss": 3.2982744547292735, + "tokens_seen": 3279814656 + }, + { + "epoch": 0.94, + "learning_rate": 6.544054747647562e-05, + "loss": 2.5144, + "theoretical_loss": 3.2982645030941065, + "tokens_seen": 3279945728 + }, + { + "epoch": 0.94, + "learning_rate": 6.501283147989735e-05, + "loss": 2.5943, + "theoretical_loss": 3.298254551967961, + "tokens_seen": 3280076800 + }, + { + "epoch": 0.94, + "learning_rate": 6.458511548331908e-05, + "loss": 2.7149, + "theoretical_loss": 3.298244601350792, + "tokens_seen": 3280207872 + }, + { + "epoch": 0.94, + "learning_rate": 6.415739948674081e-05, + "loss": 2.5036, + "theoretical_loss": 3.2982346512425518, + "tokens_seen": 3280338944 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1796054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.116804599761963, + "objective/train/theoretical_loss": 3.298229676379266, + "objective/train/tokens_used": 310455776, + "theoretical_loss": 3.298229676379266, + "tokens_seen": 3280404480 + }, + { + "epoch": 0.94, + "learning_rate": 6.372968349016254e-05, + "loss": 2.5789, + "theoretical_loss": 3.298224701643195, + "tokens_seen": 3280470016 + }, + { + "epoch": 0.94, + "learning_rate": 6.330196749358425e-05, + "loss": 2.4719, + "theoretical_loss": 3.298214752552674, + "tokens_seen": 3280601088 + }, + { + "epoch": 0.94, + "learning_rate": 6.287425149700598e-05, + "loss": 2.554, + "theoretical_loss": 3.2982048039709437, + "tokens_seen": 3280732160 + }, + { + "epoch": 0.94, + "learning_rate": 6.244653550042772e-05, + "loss": 2.5636, + "theoretical_loss": 3.2981948558979575, + "tokens_seen": 3280863232 + }, + { + "epoch": 0.94, + "learning_rate": 6.201881950384944e-05, + "loss": 2.6817, + "theoretical_loss": 3.2981849083336683, + "tokens_seen": 3280994304 + }, + { + "epoch": 0.94, + "learning_rate": 6.159110350727117e-05, + "loss": 2.578, + "theoretical_loss": 3.2981749612780304, + "tokens_seen": 3281125376 + }, + { + "epoch": 0.94, + "learning_rate": 6.11633875106929e-05, + "loss": 2.6012, + "theoretical_loss": 3.2981650147309978, + "tokens_seen": 3281256448 + }, + { + "epoch": 0.94, + "learning_rate": 6.073567151411463e-05, + "loss": 2.5616, + "theoretical_loss": 3.298155068692523, + "tokens_seen": 3281387520 + }, + { + "epoch": 0.94, + "learning_rate": 6.030795551753636e-05, + "loss": 2.4797, + "theoretical_loss": 3.2981451231625614, + "tokens_seen": 3281518592 + }, + { + "epoch": 0.94, + "learning_rate": 5.988023952095808e-05, + "loss": 2.4314, + "theoretical_loss": 3.298135178141065, + "tokens_seen": 3281649664 + }, + { + "epoch": 0.94, + "learning_rate": 5.945252352437981e-05, + "loss": 2.5998, + "theoretical_loss": 3.2981252336279887, + "tokens_seen": 3281780736 + }, + { + "epoch": 0.94, + "learning_rate": 5.902480752780154e-05, + "loss": 2.6522, + "theoretical_loss": 3.298115289623285, + "tokens_seen": 3281911808 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1796622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5396347045898438, + "objective/train/theoretical_loss": 3.298105346126909, + "objective/train/tokens_used": 312094176, + "theoretical_loss": 3.298105346126909, + "tokens_seen": 3282042880 + }, + { + "epoch": 0.94, + "learning_rate": 5.859709153122327e-05, + "loss": 2.5503, + "theoretical_loss": 3.298105346126909, + "tokens_seen": 3282042880 + }, + { + "epoch": 0.94, + "learning_rate": 5.8169375534645e-05, + "loss": 2.5614, + "theoretical_loss": 3.2980954031388134, + "tokens_seen": 3282173952 + }, + { + "epoch": 0.94, + "learning_rate": 5.774165953806673e-05, + "loss": 2.5281, + "theoretical_loss": 3.2980854606589527, + "tokens_seen": 3282305024 + }, + { + "epoch": 0.94, + "learning_rate": 5.731394354148845e-05, + "loss": 2.423, + "theoretical_loss": 3.2980755186872797, + "tokens_seen": 3282436096 + }, + { + "epoch": 0.94, + "learning_rate": 5.688622754491018e-05, + "loss": 2.6262, + "theoretical_loss": 3.298065577223749, + "tokens_seen": 3282567168 + }, + { + "epoch": 0.94, + "learning_rate": 5.645851154833191e-05, + "loss": 2.4942, + "theoretical_loss": 3.2980556362683138, + "tokens_seen": 3282698240 + }, + { + "epoch": 0.94, + "learning_rate": 5.6030795551753635e-05, + "loss": 2.5551, + "theoretical_loss": 3.298045695820928, + "tokens_seen": 3282829312 + }, + { + "epoch": 0.94, + "learning_rate": 5.5603079555175364e-05, + "loss": 2.4699, + "theoretical_loss": 3.2980357558815454, + "tokens_seen": 3282960384 + }, + { + "epoch": 0.95, + "learning_rate": 5.517536355859709e-05, + "loss": 2.4786, + "theoretical_loss": 3.29802581645012, + "tokens_seen": 3283091456 + }, + { + "epoch": 0.95, + "learning_rate": 5.474764756201882e-05, + "loss": 2.4763, + "theoretical_loss": 3.2980158775266046, + "tokens_seen": 3283222528 + }, + { + "epoch": 0.95, + "learning_rate": 5.431993156544055e-05, + "loss": 2.5492, + "theoretical_loss": 3.2980059391109546, + "tokens_seen": 3283353600 + }, + { + "epoch": 0.95, + "learning_rate": 5.389221556886228e-05, + "loss": 2.5066, + "theoretical_loss": 3.2979960012031224, + "tokens_seen": 3283484672 + }, + { + "epoch": 0.95, + "learning_rate": 5.346449957228401e-05, + "loss": 2.5815, + "theoretical_loss": 3.2979860638030623, + "tokens_seen": 3283615744 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1797896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8677234649658203, + "objective/train/theoretical_loss": 3.297981095293432, + "objective/train/tokens_used": 313732576, + "theoretical_loss": 3.297981095293432, + "tokens_seen": 3283681280 + }, + { + "epoch": 0.95, + "learning_rate": 5.303678357570573e-05, + "loss": 2.5029, + "theoretical_loss": 3.2979761269107275, + "tokens_seen": 3283746816 + }, + { + "epoch": 0.95, + "learning_rate": 5.260906757912746e-05, + "loss": 2.4992, + "theoretical_loss": 3.297966190526073, + "tokens_seen": 3283877888 + }, + { + "epoch": 0.95, + "learning_rate": 5.2181351582549194e-05, + "loss": 2.372, + "theoretical_loss": 3.2979562546490517, + "tokens_seen": 3284008960 + }, + { + "epoch": 0.95, + "learning_rate": 5.1753635585970916e-05, + "loss": 2.6307, + "theoretical_loss": 3.2979463192796175, + "tokens_seen": 3284140032 + }, + { + "epoch": 0.95, + "learning_rate": 5.1325919589392645e-05, + "loss": 2.3876, + "theoretical_loss": 3.2979363844177243, + "tokens_seen": 3284271104 + }, + { + "epoch": 0.95, + "learning_rate": 5.089820359281437e-05, + "loss": 2.4917, + "theoretical_loss": 3.297926450063326, + "tokens_seen": 3284402176 + }, + { + "epoch": 0.95, + "learning_rate": 5.0470487596236095e-05, + "loss": 2.4429, + "theoretical_loss": 3.2979165162163766, + "tokens_seen": 3284533248 + }, + { + "epoch": 0.95, + "learning_rate": 5.004277159965783e-05, + "loss": 2.5082, + "theoretical_loss": 3.2979065828768297, + "tokens_seen": 3284664320 + }, + { + "epoch": 0.95, + "learning_rate": 4.961505560307956e-05, + "loss": 2.473, + "theoretical_loss": 3.2978966500446387, + "tokens_seen": 3284795392 + }, + { + "epoch": 0.95, + "learning_rate": 4.918733960650128e-05, + "loss": 2.4112, + "theoretical_loss": 3.297886717719758, + "tokens_seen": 3284926464 + }, + { + "epoch": 0.95, + "learning_rate": 4.875962360992301e-05, + "loss": 2.3503, + "theoretical_loss": 3.2978767859021416, + "tokens_seen": 3285057536 + }, + { + "epoch": 0.95, + "learning_rate": 4.833190761334474e-05, + "loss": 2.5488, + "theoretical_loss": 3.297866854591743, + "tokens_seen": 3285188608 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1799213, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.777282953262329, + "objective/train/theoretical_loss": 3.2978569237885162, + "objective/train/tokens_used": 315370976, + "theoretical_loss": 3.2978569237885162, + "tokens_seen": 3285319680 + }, + { + "epoch": 0.95, + "learning_rate": 4.790419161676647e-05, + "loss": 2.421, + "theoretical_loss": 3.2978569237885162, + "tokens_seen": 3285319680 + }, + { + "epoch": 0.95, + "learning_rate": 4.74764756201882e-05, + "loss": 2.453, + "theoretical_loss": 3.297846993492415, + "tokens_seen": 3285450752 + }, + { + "epoch": 0.95, + "learning_rate": 4.7048759623609925e-05, + "loss": 2.5887, + "theoretical_loss": 3.297837063703393, + "tokens_seen": 3285581824 + }, + { + "epoch": 0.95, + "learning_rate": 4.662104362703165e-05, + "loss": 2.5522, + "theoretical_loss": 3.2978271344214045, + "tokens_seen": 3285712896 + }, + { + "epoch": 0.95, + "learning_rate": 4.6193327630453376e-05, + "loss": 2.6284, + "theoretical_loss": 3.2978172056464032, + "tokens_seen": 3285843968 + }, + { + "epoch": 0.95, + "learning_rate": 4.576561163387511e-05, + "loss": 2.6227, + "theoretical_loss": 3.2978072773783436, + "tokens_seen": 3285975040 + }, + { + "epoch": 0.96, + "learning_rate": 4.533789563729684e-05, + "loss": 2.5225, + "theoretical_loss": 3.2977973496171784, + "tokens_seen": 3286106112 + }, + { + "epoch": 0.96, + "learning_rate": 4.491017964071856e-05, + "loss": 2.5684, + "theoretical_loss": 3.2977874223628625, + "tokens_seen": 3286237184 + }, + { + "epoch": 0.96, + "learning_rate": 4.448246364414029e-05, + "loss": 2.5917, + "theoretical_loss": 3.297777495615349, + "tokens_seen": 3286368256 + }, + { + "epoch": 0.96, + "learning_rate": 4.405474764756202e-05, + "loss": 2.5708, + "theoretical_loss": 3.2977675693745927, + "tokens_seen": 3286499328 + } + ], + "max_steps": 2362, + "num_train_epochs": 9223372036854775807, + "total_flos": 1.51106227273728e+17, + "trial_name": null, + "trial_params": null +}