{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.09929697740000794, "global_step": 5000, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "learning_rate": 1.5873015873015873e-06, "loss": 10.5641, "theoretical_loss": 20.84128112621979, "tokens_seen": 65536 }, { "epoch": 0.0, "learning_rate": 3.1746031746031746e-06, "loss": 10.5513, "theoretical_loss": 17.594664429001284, "tokens_seen": 131072 }, { "epoch": 0.0, "learning_rate": 4.7619047619047615e-06, "loss": 10.4631, "theoretical_loss": 15.967940417509208, "tokens_seen": 196608 }, { "epoch": 0.0, "learning_rate": 6.349206349206349e-06, "loss": 10.2495, "theoretical_loss": 14.92078008066121, "tokens_seen": 262144 }, { "epoch": 0.0, "learning_rate": 7.936507936507936e-06, "loss": 10.0077, "theoretical_loss": 14.164680262435617, "tokens_seen": 327680 }, { "epoch": 0.0, "learning_rate": 9.523809523809523e-06, "loss": 9.7256, "theoretical_loss": 13.581024797222863, "tokens_seen": 393216 }, { "epoch": 0.0, "learning_rate": 1.1111111111111112e-05, "loss": 9.5124, "theoretical_loss": 13.11027232607383, "tokens_seen": 458752 }, { "epoch": 0.0, "learning_rate": 1.2698412698412699e-05, "loss": 9.3381, "theoretical_loss": 12.718592950155966, "tokens_seen": 524288 }, { "epoch": 0.0, "learning_rate": 1.4285714285714285e-05, "loss": 9.1669, "theoretical_loss": 12.385055788546264, "tokens_seen": 589824 }, { "epoch": 0.0, "learning_rate": 1.5873015873015872e-05, "loss": 8.9763, "theoretical_loss": 12.09587593170772, "tokens_seen": 655360 }, { "epoch": 0.0, "learning_rate": 1.746031746031746e-05, "loss": 8.8295, "theoretical_loss": 11.841529555774319, "tokens_seen": 720896 }, { "epoch": 0.0, "learning_rate": 1.9047619047619046e-05, "loss": 8.8699, "theoretical_loss": 11.615182533379372, "tokens_seen": 786432 }, { "epoch": 0.0, "learning_rate": 2.0634920634920636e-05, "loss": 8.7295, "theoretical_loss": 11.41177746485943, "tokens_seen": 851968 }, { "epoch": 0.0, "learning_rate": 2.2222222222222223e-05, "loss": 8.6618, "theoretical_loss": 11.227475026784514, "tokens_seen": 917504 }, { "epoch": 0.0, "learning_rate": 2.380952380952381e-05, "loss": 8.6244, "theoretical_loss": 11.059297242650555, "tokens_seen": 983040 }, { "epoch": 0.0, "learning_rate": 2.5396825396825397e-05, "loss": 8.5977, "theoretical_loss": 10.90489141112959, "tokens_seen": 1048576 }, { "epoch": 0.0, "learning_rate": 2.6984126984126984e-05, "loss": 8.4756, "theoretical_loss": 10.762369064735367, "tokens_seen": 1114112 }, { "epoch": 0.0, "learning_rate": 2.857142857142857e-05, "loss": 8.4981, "theoretical_loss": 10.63019320090292, "tokens_seen": 1179648 }, { "epoch": 0.0, "learning_rate": 3.0158730158730158e-05, "loss": 8.3172, "theoretical_loss": 10.507097497029282, "tokens_seen": 1245184 }, { "epoch": 0.0, "learning_rate": 3.1746031746031745e-05, "loss": 8.176, "theoretical_loss": 10.392027268435971, "tokens_seen": 1310720 }, { "epoch": 0.0, "learning_rate": 3.3333333333333335e-05, "loss": 8.1964, "theoretical_loss": 10.284095545569386, "tokens_seen": 1376256 }, { "epoch": 0.0, "learning_rate": 3.492063492063492e-05, "loss": 8.1379, "theoretical_loss": 10.182549877942659, "tokens_seen": 1441792 }, { "epoch": 0.0, "learning_rate": 3.650793650793651e-05, "loss": 7.9969, "theoretical_loss": 10.08674688640524, "tokens_seen": 1507328 }, { "epoch": 0.0, "learning_rate": 3.809523809523809e-05, "loss": 8.1059, "theoretical_loss": 9.99613250351292, "tokens_seen": 1572864 }, { "epoch": 0.0, "learning_rate": 3.968253968253968e-05, "loss": 7.9094, "theoretical_loss": 9.91022645106575, "tokens_seen": 1638400 }, { "epoch": 0.0, "learning_rate": 4.126984126984127e-05, "loss": 7.7211, "theoretical_loss": 9.828609916213201, "tokens_seen": 1703936 }, { "epoch": 0.0, "learning_rate": 4.2857142857142856e-05, "loss": 7.7885, "theoretical_loss": 9.750915671580087, "tokens_seen": 1769472 }, { "epoch": 0.0, "learning_rate": 4.4444444444444447e-05, "loss": 7.7711, "theoretical_loss": 9.676820083754187, "tokens_seen": 1835008 }, { "epoch": 0.0, "learning_rate": 4.603174603174603e-05, "loss": 7.6633, "theoretical_loss": 9.606036595818281, "tokens_seen": 1900544 }, { "epoch": 0.0, "learning_rate": 4.761904761904762e-05, "loss": 7.5596, "theoretical_loss": 9.538310371437493, "tokens_seen": 1966080 }, { "epoch": 0.0, "learning_rate": 4.9206349206349204e-05, "loss": 7.5956, "theoretical_loss": 9.473413862304575, "tokens_seen": 2031616 }, { "epoch": 0.0, "learning_rate": 5.0793650793650794e-05, "loss": 7.4655, "theoretical_loss": 9.411143115583098, "tokens_seen": 2097152 }, { "epoch": 0.0, "learning_rate": 5.2380952380952384e-05, "loss": 7.5066, "theoretical_loss": 9.351314678906622, "tokens_seen": 2162688 }, { "epoch": 0.0, "learning_rate": 5.396825396825397e-05, "loss": 7.3266, "theoretical_loss": 9.293762991332915, "tokens_seen": 2228224 }, { "epoch": 0.0, "learning_rate": 5.555555555555556e-05, "loss": 7.2918, "theoretical_loss": 9.238338172117398, "tokens_seen": 2293760 }, { "epoch": 0.0, "learning_rate": 5.714285714285714e-05, "loss": 7.3421, "theoretical_loss": 9.184904137180933, "tokens_seen": 2359296 }, { "epoch": 0.0, "learning_rate": 5.873015873015873e-05, "loss": 7.2943, "theoretical_loss": 9.133336987085992, "tokens_seen": 2424832 }, { "epoch": 0.0, "learning_rate": 6.0317460317460316e-05, "loss": 7.277, "theoretical_loss": 9.083523621206375, "tokens_seen": 2490368 }, { "epoch": 0.0, "learning_rate": 6.19047619047619e-05, "loss": 7.1754, "theoretical_loss": 9.03536054131647, "tokens_seen": 2555904 }, { "epoch": 0.0, "learning_rate": 6.349206349206349e-05, "loss": 7.1903, "theoretical_loss": 8.988752814581996, "tokens_seen": 2621440 }, { "epoch": 0.0, "learning_rate": 6.507936507936509e-05, "loss": 7.1207, "theoretical_loss": 8.943613171313253, "tokens_seen": 2686976 }, { "epoch": 0.0, "learning_rate": 6.666666666666667e-05, "loss": 7.0332, "theoretical_loss": 8.899861217150864, "tokens_seen": 2752512 }, { "epoch": 0.0, "learning_rate": 6.825396825396825e-05, "loss": 6.8774, "theoretical_loss": 8.85742274282614, "tokens_seen": 2818048 }, { "epoch": 0.0, "learning_rate": 6.984126984126984e-05, "loss": 6.9871, "theoretical_loss": 8.816229117451055, "tokens_seen": 2883584 }, { "epoch": 0.0, "learning_rate": 7.142857142857143e-05, "loss": 6.7777, "theoretical_loss": 8.776216753584098, "tokens_seen": 2949120 }, { "epoch": 0.0, "learning_rate": 7.301587301587302e-05, "loss": 6.8747, "theoretical_loss": 8.737326634193474, "tokens_seen": 3014656 }, { "epoch": 0.0, "learning_rate": 7.460317460317461e-05, "loss": 6.6371, "theoretical_loss": 8.699503893181488, "tokens_seen": 3080192 }, { "epoch": 0.0, "learning_rate": 7.619047619047618e-05, "loss": 6.5967, "theoretical_loss": 8.662697442408115, "tokens_seen": 3145728 }, { "epoch": 0.0, "learning_rate": 7.777777777777778e-05, "loss": 6.7174, "theoretical_loss": 8.626859639208966, "tokens_seen": 3211264 }, { "epoch": 0.0, "objective/train/docs_used": 7703, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 6.697691917419434, "objective/train/theoretical_loss": 8.591945989283708, "objective/train/tokens_used": 23736800, "theoretical_loss": 8.591945989283708, "tokens_seen": 3276800 }, { "epoch": 0.0, "learning_rate": 7.936507936507937e-05, "loss": 6.7105, "theoretical_loss": 8.591945989283708, "tokens_seen": 3276800 }, { "epoch": 0.0, "learning_rate": 8.095238095238096e-05, "loss": 6.599, "theoretical_loss": 8.557914880567758, "tokens_seen": 3342336 }, { "epoch": 0.0, "learning_rate": 8.253968253968255e-05, "loss": 6.4888, "theoretical_loss": 8.524727344318643, "tokens_seen": 3407872 }, { "epoch": 0.0, "learning_rate": 8.412698412698413e-05, "loss": 6.4092, "theoretical_loss": 8.492346840169828, "tokens_seen": 3473408 }, { "epoch": 0.0, "learning_rate": 8.571428571428571e-05, "loss": 6.5305, "theoretical_loss": 8.460739062345422, "tokens_seen": 3538944 }, { "epoch": 0.0, "learning_rate": 8.730158730158731e-05, "loss": 6.3927, "theoretical_loss": 8.429871764603359, "tokens_seen": 3604480 }, { "epoch": 0.0, "learning_rate": 8.888888888888889e-05, "loss": 6.2062, "theoretical_loss": 8.399714601792851, "tokens_seen": 3670016 }, { "epoch": 0.0, "learning_rate": 9.047619047619048e-05, "loss": 6.2861, "theoretical_loss": 8.370238986183718, "tokens_seen": 3735552 }, { "epoch": 0.0, "learning_rate": 9.206349206349206e-05, "loss": 6.2594, "theoretical_loss": 8.34141795695797, "tokens_seen": 3801088 }, { "epoch": 0.0, "learning_rate": 9.365079365079366e-05, "loss": 6.0057, "theoretical_loss": 8.313226061453822, "tokens_seen": 3866624 }, { "epoch": 0.0, "learning_rate": 9.523809523809524e-05, "loss": 6.1933, "theoretical_loss": 8.285639246924504, "tokens_seen": 3932160 }, { "epoch": 0.0, "learning_rate": 9.682539682539682e-05, "loss": 6.1101, "theoretical_loss": 8.258634761722755, "tokens_seen": 3997696 }, { "epoch": 0.0, "learning_rate": 9.841269841269841e-05, "loss": 6.0988, "theoretical_loss": 8.232191064950612, "tokens_seen": 4063232 }, { "epoch": 0.0, "learning_rate": 0.0001, "loss": 6.0056, "theoretical_loss": 8.206287743725687, "tokens_seen": 4128768 }, { "epoch": 0.0, "learning_rate": 0.00010158730158730159, "loss": 6.0406, "theoretical_loss": 8.180905437312258, "tokens_seen": 4194304 }, { "epoch": 0.0, "learning_rate": 0.00010317460317460319, "loss": 5.9078, "theoretical_loss": 8.15602576745014, "tokens_seen": 4259840 }, { "epoch": 0.0, "learning_rate": 0.00010476190476190477, "loss": 5.7249, "theoretical_loss": 8.13163127428835, "tokens_seen": 4325376 }, { "epoch": 0.0, "learning_rate": 0.00010634920634920635, "loss": 5.847, "theoretical_loss": 8.107705357395282, "tokens_seen": 4390912 }, { "epoch": 0.0, "learning_rate": 0.00010793650793650794, "loss": 5.7916, "theoretical_loss": 8.084232221374057, "tokens_seen": 4456448 }, { "epoch": 0.0, "learning_rate": 0.00010952380952380953, "loss": 5.7336, "theoretical_loss": 8.06119682566165, "tokens_seen": 4521984 }, { "epoch": 0.0, "learning_rate": 0.00011111111111111112, "loss": 5.7012, "theoretical_loss": 8.038584838134476, "tokens_seen": 4587520 }, { "epoch": 0.0, "learning_rate": 0.0001126984126984127, "loss": 5.6223, "theoretical_loss": 8.016382592182019, "tokens_seen": 4653056 }, { "epoch": 0.0, "learning_rate": 0.00011428571428571428, "loss": 5.6187, "theoretical_loss": 7.994577046944442, "tokens_seen": 4718592 }, { "epoch": 0.0, "learning_rate": 0.0001158730158730159, "loss": 5.5594, "theoretical_loss": 7.973155750440629, "tokens_seen": 4784128 }, { "epoch": 0.0, "learning_rate": 0.00011746031746031746, "loss": 5.5114, "theoretical_loss": 7.95210680534016, "tokens_seen": 4849664 }, { "epoch": 0.0, "learning_rate": 0.00011904761904761905, "loss": 5.7208, "theoretical_loss": 7.931418837156708, "tokens_seen": 4915200 }, { "epoch": 0.0, "learning_rate": 0.00012063492063492063, "loss": 5.6176, "theoretical_loss": 7.911080964661844, "tokens_seen": 4980736 }, { "epoch": 0.0, "learning_rate": 0.00012222222222222224, "loss": 5.4612, "theoretical_loss": 7.891082772337269, "tokens_seen": 5046272 }, { "epoch": 0.0, "learning_rate": 0.0001238095238095238, "loss": 5.4758, "theoretical_loss": 7.871414284700578, "tokens_seen": 5111808 }, { "epoch": 0.0, "learning_rate": 0.0001253968253968254, "loss": 5.4206, "theoretical_loss": 7.852065942354954, "tokens_seen": 5177344 }, { "epoch": 0.0, "learning_rate": 0.00012698412698412698, "loss": 5.4236, "theoretical_loss": 7.833028579626806, "tokens_seen": 5242880 }, { "epoch": 0.0, "learning_rate": 0.00012857142857142858, "loss": 5.526, "theoretical_loss": 7.814293403667714, "tokens_seen": 5308416 }, { "epoch": 0.0, "learning_rate": 0.00013015873015873017, "loss": 5.4883, "theoretical_loss": 7.795851974908005, "tokens_seen": 5373952 }, { "epoch": 0.0, "learning_rate": 0.00013174603174603174, "loss": 5.3321, "theoretical_loss": 7.77769618875926, "tokens_seen": 5439488 }, { "epoch": 0.0, "learning_rate": 0.00013333333333333334, "loss": 5.3773, "theoretical_loss": 7.7598182584719595, "tokens_seen": 5505024 }, { "epoch": 0.0, "learning_rate": 0.00013492063492063494, "loss": 5.3975, "theoretical_loss": 7.742210699062561, "tokens_seen": 5570560 }, { "epoch": 0.0, "learning_rate": 0.0001365079365079365, "loss": 5.301, "theoretical_loss": 7.724866312231575, "tokens_seen": 5636096 }, { "epoch": 0.0, "learning_rate": 0.0001380952380952381, "loss": 5.3225, "theoretical_loss": 7.707778172200822, "tokens_seen": 5701632 }, { "epoch": 0.0, "learning_rate": 0.00013968253968253967, "loss": 5.4532, "theoretical_loss": 7.6909396124039695, "tokens_seen": 5767168 }, { "epoch": 0.0, "learning_rate": 0.0001412698412698413, "loss": 5.3501, "theoretical_loss": 7.67434421296992, "tokens_seen": 5832704 }, { "epoch": 0.0, "learning_rate": 0.00014285714285714287, "loss": 5.3752, "theoretical_loss": 7.657985788943502, "tokens_seen": 5898240 }, { "epoch": 0.0, "learning_rate": 0.00014444444444444444, "loss": 5.1601, "theoretical_loss": 7.641858379192337, "tokens_seen": 5963776 }, { "epoch": 0.0, "learning_rate": 0.00014603174603174603, "loss": 5.1855, "theoretical_loss": 7.625956235952921, "tokens_seen": 6029312 }, { "epoch": 0.0, "learning_rate": 0.00014761904761904763, "loss": 5.2364, "theoretical_loss": 7.610273814972529, "tokens_seen": 6094848 }, { "epoch": 0.0, "learning_rate": 0.00014920634920634923, "loss": 5.1903, "theoretical_loss": 7.594805766207012, "tokens_seen": 6160384 }, { "epoch": 0.0, "learning_rate": 0.0001507936507936508, "loss": 5.2898, "theoretical_loss": 7.57954692503759, "tokens_seen": 6225920 }, { "epoch": 0.0, "learning_rate": 0.00015238095238095237, "loss": 5.1998, "theoretical_loss": 7.5644923039725604, "tokens_seen": 6291456 }, { "epoch": 0.0, "learning_rate": 0.000153968253968254, "loss": 4.9998, "theoretical_loss": 7.549637084802403, "tokens_seen": 6356992 }, { "epoch": 0.0, "learning_rate": 0.00015555555555555556, "loss": 5.2391, "theoretical_loss": 7.534976611179138, "tokens_seen": 6422528 }, { "epoch": 0.0, "learning_rate": 0.00015714285714285716, "loss": 5.1035, "theoretical_loss": 7.52050638159289, "tokens_seen": 6488064 }, { "epoch": 0.0, "objective/train/docs_used": 8901, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 4.883451461791992, "objective/train/theoretical_loss": 7.506222042720672, "objective/train/tokens_used": 27013600, "theoretical_loss": 7.506222042720672, "tokens_seen": 6553600 }, { "epoch": 0.0, "learning_rate": 0.00015873015873015873, "loss": 5.0335, "theoretical_loss": 7.506222042720672, "tokens_seen": 6553600 }, { "epoch": 0.0, "learning_rate": 0.00016031746031746033, "loss": 5.1059, "theoretical_loss": 7.492119383124103, "tokens_seen": 6619136 }, { "epoch": 0.0, "learning_rate": 0.00016190476190476192, "loss": 5.2313, "theoretical_loss": 7.478194327274557, "tokens_seen": 6684672 }, { "epoch": 0.0, "learning_rate": 0.0001634920634920635, "loss": 5.0888, "theoretical_loss": 7.46444292988566, "tokens_seen": 6750208 }, { "epoch": 0.0, "learning_rate": 0.0001650793650793651, "loss": 4.9131, "theoretical_loss": 7.450861370534546, "tokens_seen": 6815744 }, { "epoch": 0.0, "learning_rate": 0.0001666666666666667, "loss": 5.1414, "theoretical_loss": 7.437445948554516, "tokens_seen": 6881280 }, { "epoch": 0.0, "learning_rate": 0.00016825396825396826, "loss": 5.0188, "theoretical_loss": 7.424193078182986, "tokens_seen": 6946816 }, { "epoch": 0.0, "learning_rate": 0.00016984126984126986, "loss": 5.0327, "theoretical_loss": 7.411099283949674, "tokens_seen": 7012352 }, { "epoch": 0.0, "learning_rate": 0.00017142857142857143, "loss": 5.1783, "theoretical_loss": 7.3981611962910145, "tokens_seen": 7077888 }, { "epoch": 0.0, "learning_rate": 0.00017301587301587302, "loss": 5.1053, "theoretical_loss": 7.38537554737773, "tokens_seen": 7143424 }, { "epoch": 0.0, "learning_rate": 0.00017460317460317462, "loss": 4.9697, "theoretical_loss": 7.372739167143329, "tokens_seen": 7208960 }, { "epoch": 0.0, "learning_rate": 0.0001761904761904762, "loss": 5.009, "theoretical_loss": 7.36024897950216, "tokens_seen": 7274496 }, { "epoch": 0.0, "learning_rate": 0.00017777777777777779, "loss": 5.0185, "theoretical_loss": 7.347901998746318, "tokens_seen": 7340032 }, { "epoch": 0.0, "learning_rate": 0.00017936507936507938, "loss": 5.0807, "theoretical_loss": 7.335695326111465, "tokens_seen": 7405568 }, { "epoch": 0.0, "learning_rate": 0.00018095238095238095, "loss": 4.9311, "theoretical_loss": 7.323626146502205, "tokens_seen": 7471104 }, { "epoch": 0.0, "learning_rate": 0.00018253968253968255, "loss": 5.051, "theoretical_loss": 7.311691725368293, "tokens_seen": 7536640 }, { "epoch": 0.0, "learning_rate": 0.00018412698412698412, "loss": 5.0423, "theoretical_loss": 7.299889405723473, "tokens_seen": 7602176 }, { "epoch": 0.0, "learning_rate": 0.00018571428571428572, "loss": 4.9575, "theoretical_loss": 7.288216605299264, "tokens_seen": 7667712 }, { "epoch": 0.0, "learning_rate": 0.00018730158730158731, "loss": 4.9026, "theoretical_loss": 7.276670813826511, "tokens_seen": 7733248 }, { "epoch": 0.0, "learning_rate": 0.00018888888888888888, "loss": 4.9363, "theoretical_loss": 7.265249590437903, "tokens_seen": 7798784 }, { "epoch": 0.0, "learning_rate": 0.00019047619047619048, "loss": 4.9141, "theoretical_loss": 7.253950561185138, "tokens_seen": 7864320 }, { "epoch": 0.0, "learning_rate": 0.00019206349206349208, "loss": 4.8576, "theoretical_loss": 7.242771416664761, "tokens_seen": 7929856 }, { "epoch": 0.0, "learning_rate": 0.00019365079365079365, "loss": 4.8336, "theoretical_loss": 7.231709909747042, "tokens_seen": 7995392 }, { "epoch": 0.0, "learning_rate": 0.00019523809523809525, "loss": 4.9158, "theoretical_loss": 7.220763853402656, "tokens_seen": 8060928 }, { "epoch": 0.0, "learning_rate": 0.00019682539682539682, "loss": 4.8969, "theoretical_loss": 7.209931118622157, "tokens_seen": 8126464 }, { "epoch": 0.0, "learning_rate": 0.00019841269841269844, "loss": 5.0328, "theoretical_loss": 7.199209632423606, "tokens_seen": 8192000 }, { "epoch": 0.0, "learning_rate": 0.0002, "loss": 4.7254, "theoretical_loss": 7.188597375943913, "tokens_seen": 8257536 }, { "epoch": 0.0, "learning_rate": 0.00020158730158730158, "loss": 4.8566, "theoretical_loss": 7.178092382609779, "tokens_seen": 8323072 }, { "epoch": 0.0, "learning_rate": 0.00020317460317460318, "loss": 4.9601, "theoretical_loss": 7.1676927363842795, "tokens_seen": 8388608 }, { "epoch": 0.0, "learning_rate": 0.00020476190476190475, "loss": 4.8234, "theoretical_loss": 7.157396570085435, "tokens_seen": 8454144 }, { "epoch": 0.0, "learning_rate": 0.00020634920634920637, "loss": 4.9181, "theoretical_loss": 7.147202063773257, "tokens_seen": 8519680 }, { "epoch": 0.0, "learning_rate": 0.00020793650793650797, "loss": 4.8418, "theoretical_loss": 7.137107443201986, "tokens_seen": 8585216 }, { "epoch": 0.0, "learning_rate": 0.00020952380952380954, "loss": 4.819, "theoretical_loss": 7.127110978334411, "tokens_seen": 8650752 }, { "epoch": 0.0, "learning_rate": 0.00021111111111111113, "loss": 4.9044, "theoretical_loss": 7.117210981915334, "tokens_seen": 8716288 }, { "epoch": 0.0, "learning_rate": 0.0002126984126984127, "loss": 4.8086, "theoretical_loss": 7.10740580810139, "tokens_seen": 8781824 }, { "epoch": 0.0, "learning_rate": 0.0002142857142857143, "loss": 4.7956, "theoretical_loss": 7.097693851144607, "tokens_seen": 8847360 }, { "epoch": 0.0, "learning_rate": 0.00021587301587301587, "loss": 4.8153, "theoretical_loss": 7.088073544127209, "tokens_seen": 8912896 }, { "epoch": 0.0, "learning_rate": 0.00021746031746031744, "loss": 4.8548, "theoretical_loss": 7.07854335774531, "tokens_seen": 8978432 }, { "epoch": 0.0, "learning_rate": 0.00021904761904761907, "loss": 4.6907, "theoretical_loss": 7.069101799139266, "tokens_seen": 9043968 }, { "epoch": 0.0, "learning_rate": 0.00022063492063492066, "loss": 4.7805, "theoretical_loss": 7.0597474107685745, "tokens_seen": 9109504 }, { "epoch": 0.0, "learning_rate": 0.00022222222222222223, "loss": 4.7233, "theoretical_loss": 7.050478769329319, "tokens_seen": 9175040 }, { "epoch": 0.0, "learning_rate": 0.00022380952380952383, "loss": 4.8346, "theoretical_loss": 7.0412944847122505, "tokens_seen": 9240576 }, { "epoch": 0.0, "learning_rate": 0.0002253968253968254, "loss": 4.7503, "theoretical_loss": 7.032193198999712, "tokens_seen": 9306112 }, { "epoch": 0.0, "learning_rate": 0.000226984126984127, "loss": 4.6828, "theoretical_loss": 7.023173585499698, "tokens_seen": 9371648 }, { "epoch": 0.0, "learning_rate": 0.00022857142857142857, "loss": 4.6974, "theoretical_loss": 7.014234347815421, "tokens_seen": 9437184 }, { "epoch": 0.0, "learning_rate": 0.00023015873015873016, "loss": 4.8072, "theoretical_loss": 7.00537421894883, "tokens_seen": 9502720 }, { "epoch": 0.0, "learning_rate": 0.0002317460317460318, "loss": 4.692, "theoretical_loss": 6.996591960436656, "tokens_seen": 9568256 }, { "epoch": 0.0, "learning_rate": 0.00023333333333333336, "loss": 4.7647, "theoretical_loss": 6.9878863615175355, "tokens_seen": 9633792 }, { "epoch": 0.0, "learning_rate": 0.00023492063492063493, "loss": 4.7072, "theoretical_loss": 6.97925623832895, "tokens_seen": 9699328 }, { "epoch": 0.0, "learning_rate": 0.00023650793650793653, "loss": 4.8046, "theoretical_loss": 6.97070043313267, "tokens_seen": 9764864 }, { "epoch": 0.0, "objective/train/docs_used": 9988, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 4.5934672355651855, "objective/train/theoretical_loss": 6.962217813567541, "objective/train/tokens_used": 30290400, "theoretical_loss": 6.962217813567541, "tokens_seen": 9830400 }, { "epoch": 0.0, "learning_rate": 0.0002380952380952381, "loss": 4.7083, "theoretical_loss": 6.962217813567541, "tokens_seen": 9830400 }, { "epoch": 0.0, "learning_rate": 0.0002396825396825397, "loss": 4.6527, "theoretical_loss": 6.953807271928458, "tokens_seen": 9895936 }, { "epoch": 0.0, "learning_rate": 0.00024126984126984126, "loss": 4.4851, "theoretical_loss": 6.945467724470439, "tokens_seen": 9961472 }, { "epoch": 0.0, "learning_rate": 0.00024285714285714286, "loss": 4.4885, "theoretical_loss": 6.937198110736788, "tokens_seen": 10027008 }, { "epoch": 0.0, "learning_rate": 0.0002444444444444445, "loss": 4.5112, "theoretical_loss": 6.928997392910329, "tokens_seen": 10092544 }, { "epoch": 0.0, "learning_rate": 0.00024603174603174605, "loss": 4.7377, "theoretical_loss": 6.920864555186801, "tokens_seen": 10158080 }, { "epoch": 0.0, "learning_rate": 0.0002476190476190476, "loss": 4.6674, "theoretical_loss": 6.912798603169512, "tokens_seen": 10223616 }, { "epoch": 0.0, "learning_rate": 0.00024920634920634925, "loss": 4.5936, "theoretical_loss": 6.904798563284391, "tokens_seen": 10289152 }, { "epoch": 0.0, "learning_rate": 0.0002507936507936508, "loss": 4.8009, "theoretical_loss": 6.896863482214638, "tokens_seen": 10354688 }, { "epoch": 0.0, "learning_rate": 0.0002523809523809524, "loss": 4.7057, "theoretical_loss": 6.888992426354192, "tokens_seen": 10420224 }, { "epoch": 0.0, "learning_rate": 0.00025396825396825396, "loss": 4.7217, "theoretical_loss": 6.881184481279271, "tokens_seen": 10485760 }, { "epoch": 0.0, "learning_rate": 0.00025555555555555553, "loss": 4.5575, "theoretical_loss": 6.873438751237283, "tokens_seen": 10551296 }, { "epoch": 0.0, "learning_rate": 0.00025714285714285715, "loss": 4.6313, "theoretical_loss": 6.865754358652436, "tokens_seen": 10616832 }, { "epoch": 0.0, "learning_rate": 0.0002587301587301588, "loss": 4.5677, "theoretical_loss": 6.85813044364739, "tokens_seen": 10682368 }, { "epoch": 0.0, "learning_rate": 0.00026031746031746035, "loss": 4.5783, "theoretical_loss": 6.85056616358036, "tokens_seen": 10747904 }, { "epoch": 0.0, "learning_rate": 0.0002619047619047619, "loss": 4.7549, "theoretical_loss": 6.843060692597036, "tokens_seen": 10813440 }, { "epoch": 0.0, "learning_rate": 0.0002634920634920635, "loss": 4.5659, "theoretical_loss": 6.835613221196822, "tokens_seen": 10878976 }, { "epoch": 0.0, "learning_rate": 0.0002650793650793651, "loss": 4.6755, "theoretical_loss": 6.8282229558127865, "tokens_seen": 10944512 }, { "epoch": 0.0, "learning_rate": 0.0002666666666666667, "loss": 4.5563, "theoretical_loss": 6.820889118404866, "tokens_seen": 11010048 }, { "epoch": 0.0, "learning_rate": 0.00026825396825396825, "loss": 4.5433, "theoretical_loss": 6.813610946065789, "tokens_seen": 11075584 }, { "epoch": 0.0, "learning_rate": 0.0002698412698412699, "loss": 4.5981, "theoretical_loss": 6.806387690639279, "tokens_seen": 11141120 }, { "epoch": 0.0, "learning_rate": 0.00027142857142857144, "loss": 4.5952, "theoretical_loss": 6.799218618350064, "tokens_seen": 11206656 }, { "epoch": 0.0, "learning_rate": 0.000273015873015873, "loss": 4.6503, "theoretical_loss": 6.792103009445266, "tokens_seen": 11272192 }, { "epoch": 0.0, "learning_rate": 0.00027460317460317464, "loss": 4.4218, "theoretical_loss": 6.7850401578467645, "tokens_seen": 11337728 }, { "epoch": 0.0, "learning_rate": 0.0002761904761904762, "loss": 4.5615, "theoretical_loss": 6.778029370814127, "tokens_seen": 11403264 }, { "epoch": 0.0, "learning_rate": 0.0002777777777777778, "loss": 4.535, "theoretical_loss": 6.771069968617732, "tokens_seen": 11468800 }, { "epoch": 0.0, "learning_rate": 0.00027936507936507935, "loss": 4.6074, "theoretical_loss": 6.764161284221716, "tokens_seen": 11534336 }, { "epoch": 0.0, "learning_rate": 0.00028095238095238097, "loss": 4.7694, "theoretical_loss": 6.757302662976402, "tokens_seen": 11599872 }, { "epoch": 0.0, "learning_rate": 0.0002825396825396826, "loss": 4.5586, "theoretical_loss": 6.750493462319867, "tokens_seen": 11665408 }, { "epoch": 0.0, "learning_rate": 0.00028412698412698417, "loss": 4.4784, "theoretical_loss": 6.74373305148833, "tokens_seen": 11730944 }, { "epoch": 0.0, "learning_rate": 0.00028571428571428574, "loss": 4.4651, "theoretical_loss": 6.737020811235054, "tokens_seen": 11796480 }, { "epoch": 0.0, "learning_rate": 0.0002873015873015873, "loss": 4.3919, "theoretical_loss": 6.730356133557465, "tokens_seen": 11862016 }, { "epoch": 0.0, "learning_rate": 0.0002888888888888889, "loss": 4.5578, "theoretical_loss": 6.7237384214322, "tokens_seen": 11927552 }, { "epoch": 0.0, "learning_rate": 0.0002904761904761905, "loss": 4.4157, "theoretical_loss": 6.717167088557821, "tokens_seen": 11993088 }, { "epoch": 0.0, "learning_rate": 0.00029206349206349207, "loss": 4.534, "theoretical_loss": 6.710641559104915, "tokens_seen": 12058624 }, { "epoch": 0.0, "learning_rate": 0.00029365079365079364, "loss": 4.5115, "theoretical_loss": 6.704161267473356, "tokens_seen": 12124160 }, { "epoch": 0.0, "learning_rate": 0.00029523809523809526, "loss": 4.5263, "theoretical_loss": 6.697725658056456, "tokens_seen": 12189696 }, { "epoch": 0.0, "learning_rate": 0.00029682539682539683, "loss": 4.623, "theoretical_loss": 6.691334185011799, "tokens_seen": 12255232 }, { "epoch": 0.0, "learning_rate": 0.00029841269841269846, "loss": 4.4268, "theoretical_loss": 6.68498631203852, "tokens_seen": 12320768 }, { "epoch": 0.0, "learning_rate": 0.00030000000000000003, "loss": 4.4161, "theoretical_loss": 6.678681512160812, "tokens_seen": 12386304 }, { "epoch": 0.0, "learning_rate": 0.0003015873015873016, "loss": 4.3853, "theoretical_loss": 6.672419267517472, "tokens_seen": 12451840 }, { "epoch": 0.0, "learning_rate": 0.00030317460317460317, "loss": 4.4433, "theoretical_loss": 6.666199069157267, "tokens_seen": 12517376 }, { "epoch": 0.0, "learning_rate": 0.00030476190476190474, "loss": 4.5607, "theoretical_loss": 6.660020416839945, "tokens_seen": 12582912 }, { "epoch": 0.0, "learning_rate": 0.00030634920634920636, "loss": 4.4346, "theoretical_loss": 6.65388281884269, "tokens_seen": 12648448 }, { "epoch": 0.0, "learning_rate": 0.000307936507936508, "loss": 4.4253, "theoretical_loss": 6.647785791771864, "tokens_seen": 12713984 }, { "epoch": 0.0, "learning_rate": 0.00030952380952380956, "loss": 4.423, "theoretical_loss": 6.641728860379852, "tokens_seen": 12779520 }, { "epoch": 0.0, "learning_rate": 0.0003111111111111111, "loss": 4.5214, "theoretical_loss": 6.635711557386854, "tokens_seen": 12845056 }, { "epoch": 0.0, "learning_rate": 0.0003126984126984127, "loss": 4.5018, "theoretical_loss": 6.629733423307451, "tokens_seen": 12910592 }, { "epoch": 0.0, "learning_rate": 0.0003142857142857143, "loss": 4.5074, "theoretical_loss": 6.6237940062818215, "tokens_seen": 12976128 }, { "epoch": 0.0, "learning_rate": 0.0003158730158730159, "loss": 4.5088, "theoretical_loss": 6.617892861911426, "tokens_seen": 13041664 }, { "epoch": 0.0, "objective/train/docs_used": 11319, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 4.34834098815918, "objective/train/theoretical_loss": 6.612029553099055, "objective/train/tokens_used": 33567200, "theoretical_loss": 6.612029553099055, "tokens_seen": 13107200 }, { "epoch": 0.0, "learning_rate": 0.00031746031746031746, "loss": 4.4598, "theoretical_loss": 6.612029553099055, "tokens_seen": 13107200 }, { "epoch": 0.0, "learning_rate": 0.0003190476190476191, "loss": 4.5264, "theoretical_loss": 6.606203649893072, "tokens_seen": 13172736 }, { "epoch": 0.0, "learning_rate": 0.00032063492063492065, "loss": 4.4953, "theoretical_loss": 6.600414729335739, "tokens_seen": 13238272 }, { "epoch": 0.0, "learning_rate": 0.0003222222222222222, "loss": 4.5654, "theoretical_loss": 6.5946623753155045, "tokens_seen": 13303808 }, { "epoch": 0.0, "learning_rate": 0.00032380952380952385, "loss": 4.4299, "theoretical_loss": 6.588946178423104, "tokens_seen": 13369344 }, { "epoch": 0.0, "learning_rate": 0.0003253968253968254, "loss": 4.5707, "theoretical_loss": 6.583265735811386, "tokens_seen": 13434880 }, { "epoch": 0.0, "learning_rate": 0.000326984126984127, "loss": 4.4346, "theoretical_loss": 6.577620651058732, "tokens_seen": 13500416 }, { "epoch": 0.0, "learning_rate": 0.00032857142857142856, "loss": 4.4908, "theoretical_loss": 6.572010534035971, "tokens_seen": 13565952 }, { "epoch": 0.0, "learning_rate": 0.0003301587301587302, "loss": 4.5556, "theoretical_loss": 6.566435000776667, "tokens_seen": 13631488 }, { "epoch": 0.0, "learning_rate": 0.0003317460317460318, "loss": 4.5845, "theoretical_loss": 6.5608936733507095, "tokens_seen": 13697024 }, { "epoch": 0.0, "learning_rate": 0.0003333333333333334, "loss": 4.5322, "theoretical_loss": 6.555386179741061, "tokens_seen": 13762560 }, { "epoch": 0.0, "learning_rate": 0.00033492063492063495, "loss": 4.4791, "theoretical_loss": 6.549912153723619, "tokens_seen": 13828096 }, { "epoch": 0.0, "learning_rate": 0.0003365079365079365, "loss": 4.4018, "theoretical_loss": 6.544471234750057, "tokens_seen": 13893632 }, { "epoch": 0.0, "learning_rate": 0.0003380952380952381, "loss": 4.451, "theoretical_loss": 6.539063067833591, "tokens_seen": 13959168 }, { "epoch": 0.0, "learning_rate": 0.0003396825396825397, "loss": 4.2116, "theoretical_loss": 6.533687303437551, "tokens_seen": 14024704 }, { "epoch": 0.0, "learning_rate": 0.0003412698412698413, "loss": 4.462, "theoretical_loss": 6.528343597366709, "tokens_seen": 14090240 }, { "epoch": 0.0, "learning_rate": 0.00034285714285714285, "loss": 4.405, "theoretical_loss": 6.523031610661259, "tokens_seen": 14155776 }, { "epoch": 0.0, "learning_rate": 0.0003444444444444445, "loss": 4.5102, "theoretical_loss": 6.517751009493386, "tokens_seen": 14221312 }, { "epoch": 0.0, "learning_rate": 0.00034603174603174604, "loss": 4.3705, "theoretical_loss": 6.51250146506634, "tokens_seen": 14286848 }, { "epoch": 0.0, "learning_rate": 0.00034761904761904767, "loss": 4.4277, "theoretical_loss": 6.507282653515952, "tokens_seen": 14352384 }, { "epoch": 0.0, "learning_rate": 0.00034920634920634924, "loss": 4.384, "theoretical_loss": 6.5020942558145105, "tokens_seen": 14417920 }, { "epoch": 0.0, "learning_rate": 0.0003507936507936508, "loss": 4.4511, "theoretical_loss": 6.496935957676949, "tokens_seen": 14483456 }, { "epoch": 0.0, "learning_rate": 0.0003523809523809524, "loss": 4.4827, "theoretical_loss": 6.491807449469257, "tokens_seen": 14548992 }, { "epoch": 0.0, "learning_rate": 0.00035396825396825395, "loss": 4.4038, "theoretical_loss": 6.486708426119071, "tokens_seen": 14614528 }, { "epoch": 0.0, "learning_rate": 0.00035555555555555557, "loss": 4.4651, "theoretical_loss": 6.481638587028371, "tokens_seen": 14680064 }, { "epoch": 0.0, "learning_rate": 0.0003571428571428572, "loss": 4.2654, "theoretical_loss": 6.4765976359882345, "tokens_seen": 14745600 }, { "epoch": 0.0, "learning_rate": 0.00035873015873015877, "loss": 4.3412, "theoretical_loss": 6.47158528109558, "tokens_seen": 14811136 }, { "epoch": 0.0, "learning_rate": 0.00036031746031746034, "loss": 4.4479, "theoretical_loss": 6.466601234671855, "tokens_seen": 14876672 }, { "epoch": 0.0, "learning_rate": 0.0003619047619047619, "loss": 4.2494, "theoretical_loss": 6.461645213183605, "tokens_seen": 14942208 }, { "epoch": 0.0, "learning_rate": 0.00036349206349206353, "loss": 4.3764, "theoretical_loss": 6.45671693716488, "tokens_seen": 15007744 }, { "epoch": 0.0, "learning_rate": 0.0003650793650793651, "loss": 4.2567, "theoretical_loss": 6.451816131141427, "tokens_seen": 15073280 }, { "epoch": 0.0, "learning_rate": 0.00036666666666666667, "loss": 4.3653, "theoretical_loss": 6.4469425235566105, "tokens_seen": 15138816 }, { "epoch": 0.0, "learning_rate": 0.00036825396825396824, "loss": 4.462, "theoretical_loss": 6.442095846699032, "tokens_seen": 15204352 }, { "epoch": 0.0, "learning_rate": 0.00036984126984126986, "loss": 4.4221, "theoretical_loss": 6.437275836631781, "tokens_seen": 15269888 }, { "epoch": 0.0, "learning_rate": 0.00037142857142857143, "loss": 4.2723, "theoretical_loss": 6.432482233123297, "tokens_seen": 15335424 }, { "epoch": 0.0, "learning_rate": 0.00037301587301587306, "loss": 4.2973, "theoretical_loss": 6.427714779579773, "tokens_seen": 15400960 }, { "epoch": 0.0, "learning_rate": 0.00037460317460317463, "loss": 4.2562, "theoretical_loss": 6.422973222979093, "tokens_seen": 15466496 }, { "epoch": 0.0, "learning_rate": 0.0003761904761904762, "loss": 4.3652, "theoretical_loss": 6.418257313806215, "tokens_seen": 15532032 }, { "epoch": 0.0, "learning_rate": 0.00037777777777777777, "loss": 4.3504, "theoretical_loss": 6.413566805990028, "tokens_seen": 15597568 }, { "epoch": 0.0, "learning_rate": 0.0003793650793650794, "loss": 4.2707, "theoretical_loss": 6.408901456841564, "tokens_seen": 15663104 }, { "epoch": 0.0, "learning_rate": 0.00038095238095238096, "loss": 4.2359, "theoretical_loss": 6.404261026993607, "tokens_seen": 15728640 }, { "epoch": 0.0, "learning_rate": 0.0003825396825396826, "loss": 4.3195, "theoretical_loss": 6.399645280341607, "tokens_seen": 15794176 }, { "epoch": 0.0, "learning_rate": 0.00038412698412698416, "loss": 4.3046, "theoretical_loss": 6.395053983985889, "tokens_seen": 15859712 }, { "epoch": 0.0, "learning_rate": 0.0003857142857142857, "loss": 4.1676, "theoretical_loss": 6.390486908175121, "tokens_seen": 15925248 }, { "epoch": 0.0, "learning_rate": 0.0003873015873015873, "loss": 4.3473, "theoretical_loss": 6.385943826251012, "tokens_seen": 15990784 }, { "epoch": 0.0, "learning_rate": 0.0003888888888888889, "loss": 4.3593, "theoretical_loss": 6.381424514594194, "tokens_seen": 16056320 }, { "epoch": 0.0, "learning_rate": 0.0003904761904761905, "loss": 4.2621, "theoretical_loss": 6.376928752571271, "tokens_seen": 16121856 }, { "epoch": 0.0, "learning_rate": 0.00039206349206349206, "loss": 4.2769, "theoretical_loss": 6.372456322483007, "tokens_seen": 16187392 }, { "epoch": 0.0, "learning_rate": 0.00039365079365079363, "loss": 4.3993, "theoretical_loss": 6.368007009513611, "tokens_seen": 16252928 }, { "epoch": 0.0, "learning_rate": 0.00039523809523809526, "loss": 4.3462, "theoretical_loss": 6.363580601681101, "tokens_seen": 16318464 }, { "epoch": 0.0, "objective/train/docs_used": 12210, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 4.425621032714844, "objective/train/theoretical_loss": 6.359176889788724, "objective/train/tokens_used": 36844000, "theoretical_loss": 6.359176889788724, "tokens_seen": 16384000 }, { "epoch": 0.0, "learning_rate": 0.0003968253968253969, "loss": 4.3752, "theoretical_loss": 6.359176889788724, "tokens_seen": 16384000 }, { "epoch": 0.0, "learning_rate": 0.00039841269841269845, "loss": 4.2801, "theoretical_loss": 6.354795667377392, "tokens_seen": 16449536 }, { "epoch": 0.01, "learning_rate": 0.0004, "loss": 4.282, "theoretical_loss": 6.350436730679111, "tokens_seen": 16515072 }, { "epoch": 0.01, "learning_rate": 0.0004015873015873016, "loss": 4.3637, "theoretical_loss": 6.346099878571405, "tokens_seen": 16580608 }, { "epoch": 0.01, "learning_rate": 0.00040317460317460316, "loss": 4.2939, "theoretical_loss": 6.341784912532662, "tokens_seen": 16646144 }, { "epoch": 0.01, "learning_rate": 0.0004047619047619048, "loss": 4.3405, "theoretical_loss": 6.337491636598426, "tokens_seen": 16711680 }, { "epoch": 0.01, "learning_rate": 0.00040634920634920635, "loss": 4.1746, "theoretical_loss": 6.33321985731858, "tokens_seen": 16777216 }, { "epoch": 0.01, "learning_rate": 0.0004079365079365079, "loss": 4.2457, "theoretical_loss": 6.328969383715416, "tokens_seen": 16842752 }, { "epoch": 0.01, "learning_rate": 0.0004095238095238095, "loss": 4.254, "theoretical_loss": 6.324740027242559, "tokens_seen": 16908288 }, { "epoch": 0.01, "learning_rate": 0.0004111111111111111, "loss": 4.1043, "theoretical_loss": 6.32053160174474, "tokens_seen": 16973824 }, { "epoch": 0.01, "learning_rate": 0.00041269841269841274, "loss": 4.1842, "theoretical_loss": 6.316343923418371, "tokens_seen": 17039360 }, { "epoch": 0.01, "learning_rate": 0.00041428571428571437, "loss": 4.0588, "theoretical_loss": 6.312176810772938, "tokens_seen": 17104896 }, { "epoch": 0.01, "learning_rate": 0.00041587301587301594, "loss": 4.1163, "theoretical_loss": 6.308030084593148, "tokens_seen": 17170432 }, { "epoch": 0.01, "learning_rate": 0.0004174603174603175, "loss": 4.2286, "theoretical_loss": 6.30390356790187, "tokens_seen": 17235968 }, { "epoch": 0.01, "learning_rate": 0.0004190476190476191, "loss": 4.2368, "theoretical_loss": 6.2997970859237835, "tokens_seen": 17301504 }, { "epoch": 0.01, "learning_rate": 0.00042063492063492065, "loss": 4.2891, "theoretical_loss": 6.295710466049778, "tokens_seen": 17367040 }, { "epoch": 0.01, "learning_rate": 0.00042222222222222227, "loss": 4.3256, "theoretical_loss": 6.29164353780205, "tokens_seen": 17432576 }, { "epoch": 0.01, "learning_rate": 0.00042380952380952384, "loss": 4.2051, "theoretical_loss": 6.287596132799887, "tokens_seen": 17498112 }, { "epoch": 0.01, "learning_rate": 0.0004253968253968254, "loss": 4.2521, "theoretical_loss": 6.283568084726138, "tokens_seen": 17563648 }, { "epoch": 0.01, "learning_rate": 0.000426984126984127, "loss": 4.214, "theoretical_loss": 6.27955922929433, "tokens_seen": 17629184 }, { "epoch": 0.01, "learning_rate": 0.0004285714285714286, "loss": 4.1344, "theoretical_loss": 6.2755694042164425, "tokens_seen": 17694720 }, { "epoch": 0.01, "learning_rate": 0.0004301587301587302, "loss": 4.2665, "theoretical_loss": 6.271598449171293, "tokens_seen": 17760256 }, { "epoch": 0.01, "learning_rate": 0.00043174603174603174, "loss": 4.1745, "theoretical_loss": 6.267646205773557, "tokens_seen": 17825792 }, { "epoch": 0.01, "learning_rate": 0.0004333333333333333, "loss": 4.1578, "theoretical_loss": 6.263712517543373, "tokens_seen": 17891328 }, { "epoch": 0.01, "learning_rate": 0.0004349206349206349, "loss": 4.0887, "theoretical_loss": 6.25979722987654, "tokens_seen": 17956864 }, { "epoch": 0.01, "learning_rate": 0.0004365079365079365, "loss": 4.1966, "theoretical_loss": 6.255900190015284, "tokens_seen": 18022400 }, { "epoch": 0.01, "learning_rate": 0.00043809523809523813, "loss": 4.2064, "theoretical_loss": 6.252021247019596, "tokens_seen": 18087936 }, { "epoch": 0.01, "learning_rate": 0.00043968253968253976, "loss": 4.153, "theoretical_loss": 6.2481602517390975, "tokens_seen": 18153472 }, { "epoch": 0.01, "learning_rate": 0.0004412698412698413, "loss": 4.1275, "theoretical_loss": 6.244317056785462, "tokens_seen": 18219008 }, { "epoch": 0.01, "learning_rate": 0.0004428571428571429, "loss": 4.1322, "theoretical_loss": 6.240491516505343, "tokens_seen": 18284544 }, { "epoch": 0.01, "learning_rate": 0.00044444444444444447, "loss": 4.1679, "theoretical_loss": 6.236683486953819, "tokens_seen": 18350080 }, { "epoch": 0.01, "learning_rate": 0.0004460317460317461, "loss": 4.0966, "theoretical_loss": 6.232892825868342, "tokens_seen": 18415616 }, { "epoch": 0.01, "learning_rate": 0.00044761904761904766, "loss": 4.1138, "theoretical_loss": 6.229119392643171, "tokens_seen": 18481152 }, { "epoch": 0.01, "learning_rate": 0.00044920634920634923, "loss": 4.1271, "theoretical_loss": 6.225363048304281, "tokens_seen": 18546688 }, { "epoch": 0.01, "learning_rate": 0.0004507936507936508, "loss": 4.2432, "theoretical_loss": 6.221623655484739, "tokens_seen": 18612224 }, { "epoch": 0.01, "learning_rate": 0.00045238095238095237, "loss": 4.2264, "theoretical_loss": 6.217901078400542, "tokens_seen": 18677760 }, { "epoch": 0.01, "learning_rate": 0.000453968253968254, "loss": 4.2129, "theoretical_loss": 6.214195182826902, "tokens_seen": 18743296 }, { "epoch": 0.01, "learning_rate": 0.00045555555555555556, "loss": 4.105, "theoretical_loss": 6.2105058360749545, "tokens_seen": 18808832 }, { "epoch": 0.01, "learning_rate": 0.00045714285714285713, "loss": 4.0347, "theoretical_loss": 6.206832906968911, "tokens_seen": 18874368 }, { "epoch": 0.01, "learning_rate": 0.0004587301587301587, "loss": 4.0975, "theoretical_loss": 6.203176265823624, "tokens_seen": 18939904 }, { "epoch": 0.01, "learning_rate": 0.00046031746031746033, "loss": 4.2128, "theoretical_loss": 6.1995357844225545, "tokens_seen": 19005440 }, { "epoch": 0.01, "learning_rate": 0.0004619047619047619, "loss": 4.0438, "theoretical_loss": 6.195911335996151, "tokens_seen": 19070976 }, { "epoch": 0.01, "learning_rate": 0.0004634920634920636, "loss": 4.2097, "theoretical_loss": 6.192302795200606, "tokens_seen": 19136512 }, { "epoch": 0.01, "learning_rate": 0.00046507936507936515, "loss": 4.1158, "theoretical_loss": 6.188710038097013, "tokens_seen": 19202048 }, { "epoch": 0.01, "learning_rate": 0.0004666666666666667, "loss": 4.1666, "theoretical_loss": 6.185132942130884, "tokens_seen": 19267584 }, { "epoch": 0.01, "learning_rate": 0.0004682539682539683, "loss": 4.1723, "theoretical_loss": 6.181571386112038, "tokens_seen": 19333120 }, { "epoch": 0.01, "learning_rate": 0.00046984126984126986, "loss": 4.1159, "theoretical_loss": 6.178025250194852, "tokens_seen": 19398656 }, { "epoch": 0.01, "learning_rate": 0.0004714285714285715, "loss": 4.2241, "theoretical_loss": 6.174494415858863, "tokens_seen": 19464192 }, { "epoch": 0.01, "learning_rate": 0.00047301587301587305, "loss": 4.0897, "theoretical_loss": 6.170978765889705, "tokens_seen": 19529728 }, { "epoch": 0.01, "learning_rate": 0.0004746031746031746, "loss": 4.1264, "theoretical_loss": 6.167478184360402, "tokens_seen": 19595264 }, { "epoch": 0.01, "objective/train/docs_used": 13501, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 4.086112976074219, "objective/train/theoretical_loss": 6.163992556612967, "objective/train/tokens_used": 40120800, "theoretical_loss": 6.163992556612967, "tokens_seen": 19660800 }, { "epoch": 0.01, "learning_rate": 0.0004761904761904762, "loss": 4.1434, "theoretical_loss": 6.163992556612967, "tokens_seen": 19660800 }, { "epoch": 0.01, "learning_rate": 0.0004777777777777778, "loss": 4.0815, "theoretical_loss": 6.160521769240346, "tokens_seen": 19726336 }, { "epoch": 0.01, "learning_rate": 0.0004793650793650794, "loss": 4.1549, "theoretical_loss": 6.157065710068663, "tokens_seen": 19791872 }, { "epoch": 0.01, "learning_rate": 0.00048095238095238095, "loss": 4.0909, "theoretical_loss": 6.1536242681397795, "tokens_seen": 19857408 }, { "epoch": 0.01, "learning_rate": 0.0004825396825396825, "loss": 3.918, "theoretical_loss": 6.150197333694162, "tokens_seen": 19922944 }, { "epoch": 0.01, "learning_rate": 0.0004841269841269841, "loss": 4.0747, "theoretical_loss": 6.146784798154039, "tokens_seen": 19988480 }, { "epoch": 0.01, "learning_rate": 0.0004857142857142857, "loss": 4.1139, "theoretical_loss": 6.143386554106854, "tokens_seen": 20054016 }, { "epoch": 0.01, "learning_rate": 0.0004873015873015873, "loss": 3.9063, "theoretical_loss": 6.140002495289, "tokens_seen": 20119552 }, { "epoch": 0.01, "learning_rate": 0.000488888888888889, "loss": 4.0389, "theoretical_loss": 6.136632516569836, "tokens_seen": 20185088 }, { "epoch": 0.01, "learning_rate": 0.0004904761904761905, "loss": 4.0393, "theoretical_loss": 6.133276513935977, "tokens_seen": 20250624 }, { "epoch": 0.01, "learning_rate": 0.0004920634920634921, "loss": 4.0666, "theoretical_loss": 6.129934384475845, "tokens_seen": 20316160 }, { "epoch": 0.01, "learning_rate": 0.0004936507936507937, "loss": 4.0097, "theoretical_loss": 6.126606026364497, "tokens_seen": 20381696 }, { "epoch": 0.01, "learning_rate": 0.0004952380952380952, "loss": 4.0515, "theoretical_loss": 6.123291338848694, "tokens_seen": 20447232 }, { "epoch": 0.01, "learning_rate": 0.0004968253968253969, "loss": 4.0653, "theoretical_loss": 6.119990222232234, "tokens_seen": 20512768 }, { "epoch": 0.01, "learning_rate": 0.0004984126984126985, "loss": 4.0558, "theoretical_loss": 6.116702577861526, "tokens_seen": 20578304 }, { "epoch": 0.01, "learning_rate": 0.0005, "loss": 4.1177, "theoretical_loss": 6.113428308111404, "tokens_seen": 20643840 }, { "epoch": 0.01, "learning_rate": 0.0005015873015873016, "loss": 4.1428, "theoretical_loss": 6.110167316371188, "tokens_seen": 20709376 }, { "epoch": 0.01, "learning_rate": 0.0005031746031746032, "loss": 3.9055, "theoretical_loss": 6.106919507030966, "tokens_seen": 20774912 }, { "epoch": 0.01, "learning_rate": 0.0005047619047619048, "loss": 4.1163, "theoretical_loss": 6.103684785468113, "tokens_seen": 20840448 }, { "epoch": 0.01, "learning_rate": 0.0005063492063492064, "loss": 3.929, "theoretical_loss": 6.100463058034027, "tokens_seen": 20905984 }, { "epoch": 0.01, "learning_rate": 0.0005079365079365079, "loss": 4.0137, "theoretical_loss": 6.097254232041089, "tokens_seen": 20971520 }, { "epoch": 0.01, "learning_rate": 0.0005095238095238095, "loss": 3.9822, "theoretical_loss": 6.094058215749833, "tokens_seen": 21037056 }, { "epoch": 0.01, "learning_rate": 0.0005111111111111111, "loss": 4.1239, "theoretical_loss": 6.090874918356328, "tokens_seen": 21102592 }, { "epoch": 0.01, "learning_rate": 0.0005126984126984128, "loss": 3.888, "theoretical_loss": 6.087704249979775, "tokens_seen": 21168128 }, { "epoch": 0.01, "learning_rate": 0.0005142857142857143, "loss": 3.9305, "theoretical_loss": 6.084546121650289, "tokens_seen": 21233664 }, { "epoch": 0.01, "learning_rate": 0.0005158730158730159, "loss": 4.0623, "theoretical_loss": 6.081400445296899, "tokens_seen": 21299200 }, { "epoch": 0.01, "learning_rate": 0.0005174603174603176, "loss": 4.0524, "theoretical_loss": 6.078267133735724, "tokens_seen": 21364736 }, { "epoch": 0.01, "learning_rate": 0.0005190476190476191, "loss": 4.0179, "theoretical_loss": 6.075146100658356, "tokens_seen": 21430272 }, { "epoch": 0.01, "learning_rate": 0.0005206349206349207, "loss": 3.8274, "theoretical_loss": 6.072037260620421, "tokens_seen": 21495808 }, { "epoch": 0.01, "learning_rate": 0.0005222222222222222, "loss": 3.9728, "theoretical_loss": 6.0689405290303275, "tokens_seen": 21561344 }, { "epoch": 0.01, "learning_rate": 0.0005238095238095238, "loss": 4.0168, "theoretical_loss": 6.065855822138193, "tokens_seen": 21626880 }, { "epoch": 0.01, "learning_rate": 0.0005253968253968255, "loss": 3.9993, "theoretical_loss": 6.06278305702495, "tokens_seen": 21692416 }, { "epoch": 0.01, "learning_rate": 0.000526984126984127, "loss": 3.9977, "theoretical_loss": 6.05972215159162, "tokens_seen": 21757952 }, { "epoch": 0.01, "learning_rate": 0.0005285714285714286, "loss": 4.0765, "theoretical_loss": 6.056673024548766, "tokens_seen": 21823488 }, { "epoch": 0.01, "learning_rate": 0.0005301587301587302, "loss": 3.9494, "theoretical_loss": 6.053635595406105, "tokens_seen": 21889024 }, { "epoch": 0.01, "learning_rate": 0.0005317460317460317, "loss": 3.8884, "theoretical_loss": 6.050609784462289, "tokens_seen": 21954560 }, { "epoch": 0.01, "learning_rate": 0.0005333333333333334, "loss": 3.9952, "theoretical_loss": 6.047595512794841, "tokens_seen": 22020096 }, { "epoch": 0.01, "learning_rate": 0.0005349206349206349, "loss": 3.8465, "theoretical_loss": 6.044592702250255, "tokens_seen": 22085632 }, { "epoch": 0.01, "learning_rate": 0.0005365079365079365, "loss": 3.8355, "theoretical_loss": 6.041601275434251, "tokens_seen": 22151168 }, { "epoch": 0.01, "learning_rate": 0.0005380952380952381, "loss": 3.9425, "theoretical_loss": 6.038621155702174, "tokens_seen": 22216704 }, { "epoch": 0.01, "learning_rate": 0.0005396825396825397, "loss": 3.9988, "theoretical_loss": 6.0356522671495485, "tokens_seen": 22282240 }, { "epoch": 0.01, "learning_rate": 0.0005412698412698414, "loss": 3.9923, "theoretical_loss": 6.032694534602784, "tokens_seen": 22347776 }, { "epoch": 0.01, "learning_rate": 0.0005428571428571429, "loss": 3.8449, "theoretical_loss": 6.029747883610009, "tokens_seen": 22413312 }, { "epoch": 0.01, "learning_rate": 0.0005444444444444445, "loss": 3.8843, "theoretical_loss": 6.026812240432067, "tokens_seen": 22478848 }, { "epoch": 0.01, "learning_rate": 0.000546031746031746, "loss": 3.9525, "theoretical_loss": 6.023887532033628, "tokens_seen": 22544384 }, { "epoch": 0.01, "learning_rate": 0.0005476190476190477, "loss": 3.9851, "theoretical_loss": 6.020973686074459, "tokens_seen": 22609920 }, { "epoch": 0.01, "learning_rate": 0.0005492063492063493, "loss": 3.8682, "theoretical_loss": 6.018070630900809, "tokens_seen": 22675456 }, { "epoch": 0.01, "learning_rate": 0.0005507936507936508, "loss": 3.9546, "theoretical_loss": 6.015178295536933, "tokens_seen": 22740992 }, { "epoch": 0.01, "learning_rate": 0.0005523809523809524, "loss": 3.8417, "theoretical_loss": 6.012296609676753, "tokens_seen": 22806528 }, { "epoch": 0.01, "learning_rate": 0.0005539682539682539, "loss": 3.9796, "theoretical_loss": 6.009425503675624, "tokens_seen": 22872064 }, { "epoch": 0.01, "objective/train/docs_used": 14491, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 3.740633487701416, "objective/train/theoretical_loss": 6.006564908542249, "objective/train/tokens_used": 43397600, "theoretical_loss": 6.006564908542249, "tokens_seen": 22937600 }, { "epoch": 0.01, "learning_rate": 0.0005555555555555556, "loss": 3.8125, "theoretical_loss": 6.006564908542249, "tokens_seen": 22937600 }, { "epoch": 0.01, "learning_rate": 0.0005571428571428572, "loss": 4.0247, "theoretical_loss": 6.003714755930698, "tokens_seen": 23003136 }, { "epoch": 0.01, "learning_rate": 0.0005587301587301587, "loss": 3.8499, "theoretical_loss": 6.000874978132554, "tokens_seen": 23068672 }, { "epoch": 0.01, "learning_rate": 0.0005603174603174603, "loss": 3.7935, "theoretical_loss": 5.998045508069175, "tokens_seen": 23134208 }, { "epoch": 0.01, "learning_rate": 0.0005619047619047619, "loss": 3.6955, "theoretical_loss": 5.995226279284073, "tokens_seen": 23199744 }, { "epoch": 0.01, "learning_rate": 0.0005634920634920636, "loss": 3.848, "theoretical_loss": 5.992417225935405, "tokens_seen": 23265280 }, { "epoch": 0.01, "learning_rate": 0.0005650793650793652, "loss": 4.0086, "theoretical_loss": 5.989618282788578, "tokens_seen": 23330816 }, { "epoch": 0.01, "learning_rate": 0.0005666666666666667, "loss": 4.0299, "theoretical_loss": 5.9868293852089565, "tokens_seen": 23396352 }, { "epoch": 0.01, "learning_rate": 0.0005682539682539683, "loss": 3.7135, "theoretical_loss": 5.984050469154685, "tokens_seen": 23461888 }, { "epoch": 0.01, "learning_rate": 0.0005698412698412698, "loss": 3.9052, "theoretical_loss": 5.981281471169619, "tokens_seen": 23527424 }, { "epoch": 0.01, "learning_rate": 0.0005714285714285715, "loss": 3.916, "theoretical_loss": 5.978522328376346, "tokens_seen": 23592960 }, { "epoch": 0.01, "learning_rate": 0.0005730158730158731, "loss": 3.9566, "theoretical_loss": 5.975772978469321, "tokens_seen": 23658496 }, { "epoch": 0.01, "learning_rate": 0.0005746031746031746, "loss": 3.861, "theoretical_loss": 5.973033359708099, "tokens_seen": 23724032 }, { "epoch": 0.01, "learning_rate": 0.0005761904761904762, "loss": 4.0453, "theoretical_loss": 5.9703034109106685, "tokens_seen": 23789568 }, { "epoch": 0.01, "learning_rate": 0.0005777777777777778, "loss": 3.8675, "theoretical_loss": 5.967583071446869, "tokens_seen": 23855104 }, { "epoch": 0.01, "learning_rate": 0.0005793650793650794, "loss": 3.8817, "theoretical_loss": 5.9648722812319255, "tokens_seen": 23920640 }, { "epoch": 0.01, "learning_rate": 0.000580952380952381, "loss": 3.9538, "theoretical_loss": 5.962170980720055, "tokens_seen": 23986176 }, { "epoch": 0.01, "learning_rate": 0.0005825396825396825, "loss": 3.8526, "theoretical_loss": 5.959479110898181, "tokens_seen": 24051712 }, { "epoch": 0.01, "learning_rate": 0.0005841269841269841, "loss": 3.8755, "theoretical_loss": 5.956796613279727, "tokens_seen": 24117248 }, { "epoch": 0.01, "learning_rate": 0.0005857142857142857, "loss": 3.9102, "theoretical_loss": 5.954123429898508, "tokens_seen": 24182784 }, { "epoch": 0.01, "learning_rate": 0.0005873015873015873, "loss": 3.721, "theoretical_loss": 5.951459503302701, "tokens_seen": 24248320 }, { "epoch": 0.01, "learning_rate": 0.0005888888888888889, "loss": 3.8645, "theoretical_loss": 5.948804776548908, "tokens_seen": 24313856 }, { "epoch": 0.01, "learning_rate": 0.0005904761904761905, "loss": 4.0393, "theoretical_loss": 5.9461591931963, "tokens_seen": 24379392 }, { "epoch": 0.01, "learning_rate": 0.0005920634920634922, "loss": 3.9327, "theoretical_loss": 5.943522697300843, "tokens_seen": 24444928 }, { "epoch": 0.01, "learning_rate": 0.0005936507936507937, "loss": 3.6637, "theoretical_loss": 5.940895233409613, "tokens_seen": 24510464 }, { "epoch": 0.01, "learning_rate": 0.0005952380952380953, "loss": 3.8462, "theoretical_loss": 5.938276746555175, "tokens_seen": 24576000 }, { "epoch": 0.01, "learning_rate": 0.0005968253968253969, "loss": 3.8563, "theoretical_loss": 5.9356671822500635, "tokens_seen": 24641536 }, { "epoch": 0.01, "learning_rate": 0.0005984126984126984, "loss": 3.6734, "theoretical_loss": 5.933066486481318, "tokens_seen": 24707072 }, { "epoch": 0.01, "learning_rate": 0.0006000000000000001, "loss": 3.9587, "theoretical_loss": 5.930474605705113, "tokens_seen": 24772608 }, { "epoch": 0.01, "learning_rate": 0.0006015873015873016, "loss": 3.8129, "theoretical_loss": 5.927891486841449, "tokens_seen": 24838144 }, { "epoch": 0.01, "learning_rate": 0.0006031746031746032, "loss": 3.8233, "theoretical_loss": 5.925317077268927, "tokens_seen": 24903680 }, { "epoch": 0.01, "learning_rate": 0.0006047619047619048, "loss": 3.8697, "theoretical_loss": 5.92275132481959, "tokens_seen": 24969216 }, { "epoch": 0.01, "learning_rate": 0.0006063492063492063, "loss": 3.7977, "theoretical_loss": 5.9201941777738405, "tokens_seen": 25034752 }, { "epoch": 0.01, "learning_rate": 0.000607936507936508, "loss": 3.8119, "theoretical_loss": 5.917645584855421, "tokens_seen": 25100288 }, { "epoch": 0.01, "learning_rate": 0.0006095238095238095, "loss": 3.8489, "theoretical_loss": 5.915105495226474, "tokens_seen": 25165824 }, { "epoch": 0.01, "learning_rate": 0.0006111111111111111, "loss": 3.7896, "theoretical_loss": 5.912573858482657, "tokens_seen": 25231360 }, { "epoch": 0.01, "learning_rate": 0.0006126984126984127, "loss": 3.7893, "theoretical_loss": 5.910050624648335, "tokens_seen": 25296896 }, { "epoch": 0.01, "learning_rate": 0.0006142857142857143, "loss": 3.6833, "theoretical_loss": 5.907535744171835, "tokens_seen": 25362432 }, { "epoch": 0.01, "learning_rate": 0.000615873015873016, "loss": 3.7628, "theoretical_loss": 5.905029167920765, "tokens_seen": 25427968 }, { "epoch": 0.01, "learning_rate": 0.0006174603174603175, "loss": 3.7026, "theoretical_loss": 5.902530847177394, "tokens_seen": 25493504 }, { "epoch": 0.01, "learning_rate": 0.0006190476190476191, "loss": 3.8071, "theoretical_loss": 5.900040733634098, "tokens_seen": 25559040 }, { "epoch": 0.01, "learning_rate": 0.0006206349206349206, "loss": 3.8134, "theoretical_loss": 5.897558779388872, "tokens_seen": 25624576 }, { "epoch": 0.01, "learning_rate": 0.0006222222222222223, "loss": 3.6805, "theoretical_loss": 5.895084936940887, "tokens_seen": 25690112 }, { "epoch": 0.01, "learning_rate": 0.0006238095238095239, "loss": 3.7538, "theoretical_loss": 5.892619159186127, "tokens_seen": 25755648 }, { "epoch": 0.01, "learning_rate": 0.0006253968253968254, "loss": 3.7874, "theoretical_loss": 5.890161399413071, "tokens_seen": 25821184 }, { "epoch": 0.01, "learning_rate": 0.000626984126984127, "loss": 3.7056, "theoretical_loss": 5.887711611298428, "tokens_seen": 25886720 }, { "epoch": 0.01, "learning_rate": 0.0006285714285714286, "loss": 3.7162, "theoretical_loss": 5.885269748902955, "tokens_seen": 25952256 }, { "epoch": 0.01, "learning_rate": 0.0006301587301587302, "loss": 3.6735, "theoretical_loss": 5.882835766667295, "tokens_seen": 26017792 }, { "epoch": 0.01, "learning_rate": 0.0006317460317460318, "loss": 3.721, "theoretical_loss": 5.880409619407899, "tokens_seen": 26083328 }, { "epoch": 0.01, "learning_rate": 0.0006333333333333333, "loss": 3.618, "theoretical_loss": 5.877991262312986, "tokens_seen": 26148864 }, { "epoch": 0.01, "objective/train/docs_used": 15706, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 3.5033531188964844, "objective/train/theoretical_loss": 5.875580650938565, "objective/train/tokens_used": 46674400, "theoretical_loss": 5.875580650938565, "tokens_seen": 26214400 }, { "epoch": 0.01, "learning_rate": 0.0006349206349206349, "loss": 3.6113, "theoretical_loss": 5.875580650938565, "tokens_seen": 26214400 }, { "epoch": 0.01, "learning_rate": 0.0006365079365079365, "loss": 3.7019, "theoretical_loss": 5.873177741204501, "tokens_seen": 26279936 }, { "epoch": 0.01, "learning_rate": 0.0006380952380952382, "loss": 3.644, "theoretical_loss": 5.8707824893906455, "tokens_seen": 26345472 }, { "epoch": 0.01, "learning_rate": 0.0006396825396825398, "loss": 3.6657, "theoretical_loss": 5.868394852133001, "tokens_seen": 26411008 }, { "epoch": 0.01, "learning_rate": 0.0006412698412698413, "loss": 3.7231, "theoretical_loss": 5.866014786419951, "tokens_seen": 26476544 }, { "epoch": 0.01, "learning_rate": 0.0006428571428571429, "loss": 3.8293, "theoretical_loss": 5.86364224958853, "tokens_seen": 26542080 }, { "epoch": 0.01, "learning_rate": 0.0006444444444444444, "loss": 3.7392, "theoretical_loss": 5.861277199320743, "tokens_seen": 26607616 }, { "epoch": 0.01, "learning_rate": 0.0006460317460317461, "loss": 3.7438, "theoretical_loss": 5.858919593639941, "tokens_seen": 26673152 }, { "epoch": 0.01, "learning_rate": 0.0006476190476190477, "loss": 3.7997, "theoretical_loss": 5.856569390907229, "tokens_seen": 26738688 }, { "epoch": 0.01, "learning_rate": 0.0006492063492063492, "loss": 3.8363, "theoretical_loss": 5.854226549817936, "tokens_seen": 26804224 }, { "epoch": 0.01, "learning_rate": 0.0006507936507936508, "loss": 3.6657, "theoretical_loss": 5.851891029398114, "tokens_seen": 26869760 }, { "epoch": 0.01, "learning_rate": 0.0006523809523809524, "loss": 3.7011, "theoretical_loss": 5.8495627890011015, "tokens_seen": 26935296 }, { "epoch": 0.01, "learning_rate": 0.000653968253968254, "loss": 3.7339, "theoretical_loss": 5.847241788304114, "tokens_seen": 27000832 }, { "epoch": 0.01, "learning_rate": 0.0006555555555555556, "loss": 3.77, "theoretical_loss": 5.844927987304884, "tokens_seen": 27066368 }, { "epoch": 0.01, "learning_rate": 0.0006571428571428571, "loss": 3.5868, "theoretical_loss": 5.842621346318348, "tokens_seen": 27131904 }, { "epoch": 0.01, "learning_rate": 0.0006587301587301587, "loss": 3.709, "theoretical_loss": 5.84032182597337, "tokens_seen": 27197440 }, { "epoch": 0.01, "learning_rate": 0.0006603174603174604, "loss": 3.6199, "theoretical_loss": 5.8380293872095095, "tokens_seen": 27262976 }, { "epoch": 0.01, "learning_rate": 0.0006619047619047619, "loss": 3.6197, "theoretical_loss": 5.835743991273832, "tokens_seen": 27328512 }, { "epoch": 0.01, "learning_rate": 0.0006634920634920636, "loss": 3.507, "theoretical_loss": 5.833465599717752, "tokens_seen": 27394048 }, { "epoch": 0.01, "learning_rate": 0.0006650793650793651, "loss": 3.7224, "theoretical_loss": 5.831194174393929, "tokens_seen": 27459584 }, { "epoch": 0.01, "learning_rate": 0.0006666666666666668, "loss": 3.5851, "theoretical_loss": 5.828929677453187, "tokens_seen": 27525120 }, { "epoch": 0.01, "learning_rate": 0.0006682539682539683, "loss": 3.5854, "theoretical_loss": 5.826672071341488, "tokens_seen": 27590656 }, { "epoch": 0.01, "learning_rate": 0.0006698412698412699, "loss": 3.6313, "theoretical_loss": 5.824421318796932, "tokens_seen": 27656192 }, { "epoch": 0.01, "learning_rate": 0.0006714285714285715, "loss": 3.6213, "theoretical_loss": 5.822177382846799, "tokens_seen": 27721728 }, { "epoch": 0.01, "learning_rate": 0.000673015873015873, "loss": 3.6204, "theoretical_loss": 5.8199402268046265, "tokens_seen": 27787264 }, { "epoch": 0.01, "learning_rate": 0.0006746031746031747, "loss": 3.6112, "theoretical_loss": 5.817709814267329, "tokens_seen": 27852800 }, { "epoch": 0.01, "learning_rate": 0.0006761904761904762, "loss": 3.7146, "theoretical_loss": 5.815486109112342, "tokens_seen": 27918336 }, { "epoch": 0.01, "learning_rate": 0.0006777777777777778, "loss": 3.5925, "theoretical_loss": 5.813269075494812, "tokens_seen": 27983872 }, { "epoch": 0.01, "learning_rate": 0.0006793650793650794, "loss": 3.5826, "theoretical_loss": 5.811058677844817, "tokens_seen": 28049408 }, { "epoch": 0.01, "learning_rate": 0.0006809523809523809, "loss": 3.6308, "theoretical_loss": 5.808854880864618, "tokens_seen": 28114944 }, { "epoch": 0.01, "learning_rate": 0.0006825396825396826, "loss": 3.5636, "theoretical_loss": 5.8066576495259525, "tokens_seen": 28180480 }, { "epoch": 0.01, "learning_rate": 0.0006841269841269841, "loss": 3.5684, "theoretical_loss": 5.804466949067352, "tokens_seen": 28246016 }, { "epoch": 0.01, "learning_rate": 0.0006857142857142857, "loss": 3.6715, "theoretical_loss": 5.8022827449914995, "tokens_seen": 28311552 }, { "epoch": 0.01, "learning_rate": 0.0006873015873015873, "loss": 3.6888, "theoretical_loss": 5.800105003062612, "tokens_seen": 28377088 }, { "epoch": 0.01, "learning_rate": 0.000688888888888889, "loss": 3.5941, "theoretical_loss": 5.797933689303866, "tokens_seen": 28442624 }, { "epoch": 0.01, "learning_rate": 0.0006904761904761906, "loss": 3.6386, "theoretical_loss": 5.795768769994842, "tokens_seen": 28508160 }, { "epoch": 0.01, "learning_rate": 0.0006920634920634921, "loss": 3.5843, "theoretical_loss": 5.793610211669004, "tokens_seen": 28573696 }, { "epoch": 0.01, "learning_rate": 0.0006936507936507937, "loss": 3.5697, "theoretical_loss": 5.79145798111122, "tokens_seen": 28639232 }, { "epoch": 0.01, "learning_rate": 0.0006952380952380953, "loss": 3.6252, "theoretical_loss": 5.789312045355292, "tokens_seen": 28704768 }, { "epoch": 0.01, "learning_rate": 0.0006968253968253969, "loss": 3.6311, "theoretical_loss": 5.787172371681535, "tokens_seen": 28770304 }, { "epoch": 0.01, "learning_rate": 0.0006984126984126985, "loss": 3.7606, "theoretical_loss": 5.785038927614373, "tokens_seen": 28835840 }, { "epoch": 0.01, "learning_rate": 0.0007, "loss": 3.3781, "theoretical_loss": 5.782911680919971, "tokens_seen": 28901376 }, { "epoch": 0.01, "learning_rate": 0.0007015873015873016, "loss": 3.6269, "theoretical_loss": 5.78079059960389, "tokens_seen": 28966912 }, { "epoch": 0.01, "learning_rate": 0.0007031746031746032, "loss": 3.5331, "theoretical_loss": 5.77867565190877, "tokens_seen": 29032448 }, { "epoch": 0.01, "learning_rate": 0.0007047619047619048, "loss": 3.5922, "theoretical_loss": 5.776566806312052, "tokens_seen": 29097984 }, { "epoch": 0.01, "learning_rate": 0.0007063492063492064, "loss": 3.572, "theoretical_loss": 5.774464031523705, "tokens_seen": 29163520 }, { "epoch": 0.01, "learning_rate": 0.0007079365079365079, "loss": 3.5933, "theoretical_loss": 5.772367296484001, "tokens_seen": 29229056 }, { "epoch": 0.01, "learning_rate": 0.0007095238095238095, "loss": 3.624, "theoretical_loss": 5.770276570361309, "tokens_seen": 29294592 }, { "epoch": 0.01, "learning_rate": 0.0007111111111111111, "loss": 3.5538, "theoretical_loss": 5.768191822549908, "tokens_seen": 29360128 }, { "epoch": 0.01, "learning_rate": 0.0007126984126984128, "loss": 3.5156, "theoretical_loss": 5.766113022667838, "tokens_seen": 29425664 }, { "epoch": 0.01, "objective/train/docs_used": 16666, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 3.571195363998413, "objective/train/theoretical_loss": 5.764040140554766, "objective/train/tokens_used": 49951200, "theoretical_loss": 5.764040140554766, "tokens_seen": 29491200 }, { "epoch": 0.01, "learning_rate": 0.0007142857142857144, "loss": 3.5886, "theoretical_loss": 5.764040140554766, "tokens_seen": 29491200 }, { "epoch": 0.01, "learning_rate": 0.0007158730158730159, "loss": 3.6281, "theoretical_loss": 5.761973146269881, "tokens_seen": 29556736 }, { "epoch": 0.01, "learning_rate": 0.0007174603174603175, "loss": 3.3317, "theoretical_loss": 5.759912010089819, "tokens_seen": 29622272 }, { "epoch": 0.01, "learning_rate": 0.000719047619047619, "loss": 3.5542, "theoretical_loss": 5.757856702506597, "tokens_seen": 29687808 }, { "epoch": 0.01, "learning_rate": 0.0007206349206349207, "loss": 3.584, "theoretical_loss": 5.755807194225595, "tokens_seen": 29753344 }, { "epoch": 0.01, "learning_rate": 0.0007222222222222223, "loss": 3.4624, "theoretical_loss": 5.753763456163529, "tokens_seen": 29818880 }, { "epoch": 0.01, "learning_rate": 0.0007238095238095238, "loss": 3.5575, "theoretical_loss": 5.751725459446487, "tokens_seen": 29884416 }, { "epoch": 0.01, "learning_rate": 0.0007253968253968254, "loss": 3.5754, "theoretical_loss": 5.7496931754079466, "tokens_seen": 29949952 }, { "epoch": 0.01, "learning_rate": 0.0007269841269841271, "loss": 3.6041, "theoretical_loss": 5.747666575586849, "tokens_seen": 30015488 }, { "epoch": 0.01, "learning_rate": 0.0007285714285714286, "loss": 3.5614, "theoretical_loss": 5.7456456317256785, "tokens_seen": 30081024 }, { "epoch": 0.01, "learning_rate": 0.0007301587301587302, "loss": 3.68, "theoretical_loss": 5.743630315768563, "tokens_seen": 30146560 }, { "epoch": 0.01, "learning_rate": 0.0007317460317460317, "loss": 3.5685, "theoretical_loss": 5.741620599859402, "tokens_seen": 30212096 }, { "epoch": 0.01, "learning_rate": 0.0007333333333333333, "loss": 3.646, "theoretical_loss": 5.739616456340021, "tokens_seen": 30277632 }, { "epoch": 0.01, "learning_rate": 0.000734920634920635, "loss": 3.5125, "theoretical_loss": 5.737617857748329, "tokens_seen": 30343168 }, { "epoch": 0.01, "learning_rate": 0.0007365079365079365, "loss": 3.5059, "theoretical_loss": 5.73562477681652, "tokens_seen": 30408704 }, { "epoch": 0.01, "learning_rate": 0.0007380952380952382, "loss": 3.6117, "theoretical_loss": 5.733637186469274, "tokens_seen": 30474240 }, { "epoch": 0.01, "learning_rate": 0.0007396825396825397, "loss": 3.5285, "theoretical_loss": 5.731655059821993, "tokens_seen": 30539776 }, { "epoch": 0.01, "learning_rate": 0.0007412698412698414, "loss": 3.5112, "theoretical_loss": 5.729678370179052, "tokens_seen": 30605312 }, { "epoch": 0.01, "learning_rate": 0.0007428571428571429, "loss": 3.3423, "theoretical_loss": 5.727707091032062, "tokens_seen": 30670848 }, { "epoch": 0.01, "learning_rate": 0.0007444444444444445, "loss": 3.5549, "theoretical_loss": 5.725741196058175, "tokens_seen": 30736384 }, { "epoch": 0.01, "learning_rate": 0.0007460317460317461, "loss": 3.5069, "theoretical_loss": 5.723780659118377, "tokens_seen": 30801920 }, { "epoch": 0.01, "learning_rate": 0.0007476190476190476, "loss": 3.5296, "theoretical_loss": 5.721825454255825, "tokens_seen": 30867456 }, { "epoch": 0.01, "learning_rate": 0.0007492063492063493, "loss": 3.6073, "theoretical_loss": 5.71987555569419, "tokens_seen": 30932992 }, { "epoch": 0.01, "learning_rate": 0.0007507936507936508, "loss": 3.4977, "theoretical_loss": 5.717930937836029, "tokens_seen": 30998528 }, { "epoch": 0.01, "learning_rate": 0.0007523809523809524, "loss": 3.5041, "theoretical_loss": 5.715991575261159, "tokens_seen": 31064064 }, { "epoch": 0.01, "learning_rate": 0.000753968253968254, "loss": 3.4418, "theoretical_loss": 5.714057442725068, "tokens_seen": 31129600 }, { "epoch": 0.01, "learning_rate": 0.0007555555555555555, "loss": 3.5898, "theoretical_loss": 5.712128515157324, "tokens_seen": 31195136 }, { "epoch": 0.01, "learning_rate": 0.0007571428571428572, "loss": 3.5238, "theoretical_loss": 5.710204767660022, "tokens_seen": 31260672 }, { "epoch": 0.01, "learning_rate": 0.0007587301587301588, "loss": 3.4737, "theoretical_loss": 5.708286175506232, "tokens_seen": 31326208 }, { "epoch": 0.01, "learning_rate": 0.0007603174603174603, "loss": 3.5078, "theoretical_loss": 5.706372714138472, "tokens_seen": 31391744 }, { "epoch": 0.01, "learning_rate": 0.0007619047619047619, "loss": 3.3246, "theoretical_loss": 5.704464359167195, "tokens_seen": 31457280 }, { "epoch": 0.01, "learning_rate": 0.0007634920634920635, "loss": 3.5155, "theoretical_loss": 5.702561086369294, "tokens_seen": 31522816 }, { "epoch": 0.01, "learning_rate": 0.0007650793650793652, "loss": 3.6084, "theoretical_loss": 5.7006628716866246, "tokens_seen": 31588352 }, { "epoch": 0.01, "learning_rate": 0.0007666666666666667, "loss": 3.54, "theoretical_loss": 5.698769691224539, "tokens_seen": 31653888 }, { "epoch": 0.01, "learning_rate": 0.0007682539682539683, "loss": 3.5811, "theoretical_loss": 5.696881521250441, "tokens_seen": 31719424 }, { "epoch": 0.01, "learning_rate": 0.0007698412698412699, "loss": 3.4774, "theoretical_loss": 5.694998338192356, "tokens_seen": 31784960 }, { "epoch": 0.01, "learning_rate": 0.0007714285714285715, "loss": 3.3984, "theoretical_loss": 5.693120118637513, "tokens_seen": 31850496 }, { "epoch": 0.01, "learning_rate": 0.0007730158730158731, "loss": 3.495, "theoretical_loss": 5.691246839330946, "tokens_seen": 31916032 }, { "epoch": 0.01, "learning_rate": 0.0007746031746031746, "loss": 3.3938, "theoretical_loss": 5.689378477174106, "tokens_seen": 31981568 }, { "epoch": 0.01, "learning_rate": 0.0007761904761904762, "loss": 3.4542, "theoretical_loss": 5.687515009223498, "tokens_seen": 32047104 }, { "epoch": 0.01, "learning_rate": 0.0007777777777777778, "loss": 3.5173, "theoretical_loss": 5.685656412689318, "tokens_seen": 32112640 }, { "epoch": 0.01, "learning_rate": 0.0007793650793650794, "loss": 3.4633, "theoretical_loss": 5.683802664934115, "tokens_seen": 32178176 }, { "epoch": 0.01, "learning_rate": 0.000780952380952381, "loss": 3.3623, "theoretical_loss": 5.681953743471466, "tokens_seen": 32243712 }, { "epoch": 0.01, "learning_rate": 0.0007825396825396825, "loss": 3.5352, "theoretical_loss": 5.680109625964663, "tokens_seen": 32309248 }, { "epoch": 0.01, "learning_rate": 0.0007841269841269841, "loss": 3.5366, "theoretical_loss": 5.678270290225415, "tokens_seen": 32374784 }, { "epoch": 0.01, "learning_rate": 0.0007857142857142857, "loss": 3.4648, "theoretical_loss": 5.676435714212562, "tokens_seen": 32440320 }, { "epoch": 0.01, "learning_rate": 0.0007873015873015873, "loss": 3.3286, "theoretical_loss": 5.674605876030808, "tokens_seen": 32505856 }, { "epoch": 0.01, "learning_rate": 0.000788888888888889, "loss": 3.4303, "theoretical_loss": 5.672780753929461, "tokens_seen": 32571392 }, { "epoch": 0.01, "learning_rate": 0.0007904761904761905, "loss": 3.4569, "theoretical_loss": 5.67096032630119, "tokens_seen": 32636928 }, { "epoch": 0.01, "learning_rate": 0.0007920634920634921, "loss": 3.4918, "theoretical_loss": 5.669144571680797, "tokens_seen": 32702464 }, { "epoch": 0.01, "objective/train/docs_used": 17795, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 3.261939525604248, "objective/train/theoretical_loss": 5.667333468743994, "objective/train/tokens_used": 53228000, "theoretical_loss": 5.667333468743994, "tokens_seen": 32768000 }, { "epoch": 0.01, "learning_rate": 0.0007936507936507938, "loss": 3.3689, "theoretical_loss": 5.667333468743994, "tokens_seen": 32768000 }, { "epoch": 0.01, "learning_rate": 0.0007952380952380953, "loss": 3.3711, "theoretical_loss": 5.665526996306205, "tokens_seen": 32833536 }, { "epoch": 0.01, "learning_rate": 0.0007968253968253969, "loss": 3.5329, "theoretical_loss": 5.663725133321369, "tokens_seen": 32899072 }, { "epoch": 0.01, "learning_rate": 0.0007984126984126984, "loss": 3.5592, "theoretical_loss": 5.661927858880761, "tokens_seen": 32964608 }, { "epoch": 0.01, "learning_rate": 0.0008, "loss": 3.3173, "theoretical_loss": 5.6601351522118275, "tokens_seen": 33030144 }, { "epoch": 0.01, "learning_rate": 0.0007999839518555668, "loss": 3.5531, "theoretical_loss": 5.65834699267703, "tokens_seen": 33095680 }, { "epoch": 0.01, "learning_rate": 0.0007999679037111334, "loss": 3.2348, "theoretical_loss": 5.6565633597727025, "tokens_seen": 33161216 }, { "epoch": 0.01, "learning_rate": 0.0007999518555667002, "loss": 3.4544, "theoretical_loss": 5.65478423312792, "tokens_seen": 33226752 }, { "epoch": 0.01, "learning_rate": 0.0007999358074222668, "loss": 3.4522, "theoretical_loss": 5.65300959250338, "tokens_seen": 33292288 }, { "epoch": 0.01, "learning_rate": 0.0007999197592778336, "loss": 3.4293, "theoretical_loss": 5.651239417790293, "tokens_seen": 33357824 }, { "epoch": 0.01, "learning_rate": 0.0007999037111334002, "loss": 3.3407, "theoretical_loss": 5.649473689009292, "tokens_seen": 33423360 }, { "epoch": 0.01, "learning_rate": 0.000799887662988967, "loss": 3.3672, "theoretical_loss": 5.6477123863093395, "tokens_seen": 33488896 }, { "epoch": 0.01, "learning_rate": 0.0007998716148445336, "loss": 3.403, "theoretical_loss": 5.645955489966661, "tokens_seen": 33554432 }, { "epoch": 0.01, "learning_rate": 0.0007998555667001004, "loss": 3.3867, "theoretical_loss": 5.644202980383678, "tokens_seen": 33619968 }, { "epoch": 0.01, "learning_rate": 0.000799839518555667, "loss": 3.351, "theoretical_loss": 5.6424548380879616, "tokens_seen": 33685504 }, { "epoch": 0.01, "learning_rate": 0.0007998234704112338, "loss": 3.4848, "theoretical_loss": 5.640711043731184, "tokens_seen": 33751040 }, { "epoch": 0.01, "learning_rate": 0.0007998074222668004, "loss": 3.4619, "theoretical_loss": 5.6389715780880945, "tokens_seen": 33816576 }, { "epoch": 0.01, "learning_rate": 0.0007997913741223672, "loss": 3.4355, "theoretical_loss": 5.6372364220555, "tokens_seen": 33882112 }, { "epoch": 0.01, "learning_rate": 0.0007997753259779338, "loss": 3.4325, "theoretical_loss": 5.635505556651251, "tokens_seen": 33947648 }, { "epoch": 0.01, "learning_rate": 0.0007997592778335006, "loss": 3.4346, "theoretical_loss": 5.633778963013247, "tokens_seen": 34013184 }, { "epoch": 0.01, "learning_rate": 0.0007997432296890672, "loss": 3.3384, "theoretical_loss": 5.632056622398448, "tokens_seen": 34078720 }, { "epoch": 0.01, "learning_rate": 0.000799727181544634, "loss": 3.4516, "theoretical_loss": 5.63033851618189, "tokens_seen": 34144256 }, { "epoch": 0.01, "learning_rate": 0.0007997111334002006, "loss": 3.2575, "theoretical_loss": 5.628624625855725, "tokens_seen": 34209792 }, { "epoch": 0.01, "learning_rate": 0.0007996950852557674, "loss": 3.3118, "theoretical_loss": 5.626914933028255, "tokens_seen": 34275328 }, { "epoch": 0.01, "learning_rate": 0.000799679037111334, "loss": 3.3996, "theoretical_loss": 5.625209419422983, "tokens_seen": 34340864 }, { "epoch": 0.01, "learning_rate": 0.0007996629889669008, "loss": 3.4081, "theoretical_loss": 5.623508066877678, "tokens_seen": 34406400 }, { "epoch": 0.01, "learning_rate": 0.0007996469408224675, "loss": 3.3567, "theoretical_loss": 5.621810857343441, "tokens_seen": 34471936 }, { "epoch": 0.01, "learning_rate": 0.0007996308926780342, "loss": 3.2694, "theoretical_loss": 5.620117772883784, "tokens_seen": 34537472 }, { "epoch": 0.01, "learning_rate": 0.0007996148445336009, "loss": 3.4846, "theoretical_loss": 5.618428795673718, "tokens_seen": 34603008 }, { "epoch": 0.01, "learning_rate": 0.0007995987963891676, "loss": 3.4082, "theoretical_loss": 5.616743907998853, "tokens_seen": 34668544 }, { "epoch": 0.01, "learning_rate": 0.0007995827482447343, "loss": 3.3935, "theoretical_loss": 5.6150630922545, "tokens_seen": 34734080 }, { "epoch": 0.01, "learning_rate": 0.000799566700100301, "loss": 3.3234, "theoretical_loss": 5.613386330944792, "tokens_seen": 34799616 }, { "epoch": 0.01, "learning_rate": 0.0007995506519558677, "loss": 3.3006, "theoretical_loss": 5.6117136066818, "tokens_seen": 34865152 }, { "epoch": 0.01, "learning_rate": 0.0007995346038114344, "loss": 3.3978, "theoretical_loss": 5.610044902184672, "tokens_seen": 34930688 }, { "epoch": 0.01, "learning_rate": 0.0007995185556670011, "loss": 3.3917, "theoretical_loss": 5.608380200278775, "tokens_seen": 34996224 }, { "epoch": 0.01, "learning_rate": 0.0007995025075225678, "loss": 3.2776, "theoretical_loss": 5.606719483894837, "tokens_seen": 35061760 }, { "epoch": 0.01, "learning_rate": 0.0007994864593781345, "loss": 3.429, "theoretical_loss": 5.605062736068113, "tokens_seen": 35127296 }, { "epoch": 0.01, "learning_rate": 0.0007994704112337012, "loss": 3.3658, "theoretical_loss": 5.603409939937547, "tokens_seen": 35192832 }, { "epoch": 0.01, "learning_rate": 0.0007994543630892679, "loss": 3.3409, "theoretical_loss": 5.601761078744952, "tokens_seen": 35258368 }, { "epoch": 0.01, "learning_rate": 0.0007994383149448346, "loss": 3.3312, "theoretical_loss": 5.600116135834183, "tokens_seen": 35323904 }, { "epoch": 0.01, "learning_rate": 0.0007994222668004013, "loss": 3.1932, "theoretical_loss": 5.598475094650334, "tokens_seen": 35389440 }, { "epoch": 0.01, "learning_rate": 0.000799406218655968, "loss": 3.2632, "theoretical_loss": 5.59683793873894, "tokens_seen": 35454976 }, { "epoch": 0.01, "learning_rate": 0.0007993901705115347, "loss": 3.3286, "theoretical_loss": 5.595204651745176, "tokens_seen": 35520512 }, { "epoch": 0.01, "learning_rate": 0.0007993741223671014, "loss": 3.1714, "theoretical_loss": 5.593575217413074, "tokens_seen": 35586048 }, { "epoch": 0.01, "learning_rate": 0.0007993580742226681, "loss": 3.2249, "theoretical_loss": 5.591949619584746, "tokens_seen": 35651584 }, { "epoch": 0.01, "learning_rate": 0.0007993420260782347, "loss": 3.1943, "theoretical_loss": 5.590327842199611, "tokens_seen": 35717120 }, { "epoch": 0.01, "learning_rate": 0.0007993259779338015, "loss": 3.261, "theoretical_loss": 5.588709869293634, "tokens_seen": 35782656 }, { "epoch": 0.01, "learning_rate": 0.0007993099297893681, "loss": 3.3474, "theoretical_loss": 5.5870956849985705, "tokens_seen": 35848192 }, { "epoch": 0.01, "learning_rate": 0.0007992938816449349, "loss": 3.4015, "theoretical_loss": 5.585485273541212, "tokens_seen": 35913728 }, { "epoch": 0.01, "learning_rate": 0.0007992778335005015, "loss": 3.3678, "theoretical_loss": 5.583878619242652, "tokens_seen": 35979264 }, { "epoch": 0.01, "objective/train/docs_used": 18676, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 3.2103042602539062, "objective/train/theoretical_loss": 5.582275706517549, "objective/train/tokens_used": 56504800, "theoretical_loss": 5.582275706517549, "tokens_seen": 36044800 }, { "epoch": 0.01, "learning_rate": 0.0007992617853560683, "loss": 3.3616, "theoretical_loss": 5.582275706517549, "tokens_seen": 36044800 }, { "epoch": 0.01, "learning_rate": 0.0007992457372116349, "loss": 3.3332, "theoretical_loss": 5.580676519873396, "tokens_seen": 36110336 }, { "epoch": 0.01, "learning_rate": 0.0007992296890672017, "loss": 3.5113, "theoretical_loss": 5.579081043909808, "tokens_seen": 36175872 }, { "epoch": 0.01, "learning_rate": 0.0007992136409227683, "loss": 3.4334, "theoretical_loss": 5.577489263317796, "tokens_seen": 36241408 }, { "epoch": 0.01, "learning_rate": 0.000799197592778335, "loss": 3.3125, "theoretical_loss": 5.5759011628790764, "tokens_seen": 36306944 }, { "epoch": 0.01, "learning_rate": 0.0007991815446339017, "loss": 3.4013, "theoretical_loss": 5.574316727465358, "tokens_seen": 36372480 }, { "epoch": 0.01, "learning_rate": 0.0007991654964894684, "loss": 3.2732, "theoretical_loss": 5.572735942037655, "tokens_seen": 36438016 }, { "epoch": 0.01, "learning_rate": 0.0007991494483450351, "loss": 3.1625, "theoretical_loss": 5.571158791645597, "tokens_seen": 36503552 }, { "epoch": 0.01, "learning_rate": 0.0007991334002006018, "loss": 3.284, "theoretical_loss": 5.569585261426752, "tokens_seen": 36569088 }, { "epoch": 0.01, "learning_rate": 0.0007991173520561685, "loss": 3.4507, "theoretical_loss": 5.568015336605953, "tokens_seen": 36634624 }, { "epoch": 0.01, "learning_rate": 0.0007991013039117352, "loss": 3.3153, "theoretical_loss": 5.566449002494627, "tokens_seen": 36700160 }, { "epoch": 0.01, "learning_rate": 0.0007990852557673019, "loss": 3.135, "theoretical_loss": 5.564886244490136, "tokens_seen": 36765696 }, { "epoch": 0.01, "learning_rate": 0.0007990692076228686, "loss": 3.3133, "theoretical_loss": 5.563327048075124, "tokens_seen": 36831232 }, { "epoch": 0.01, "learning_rate": 0.0007990531594784353, "loss": 3.3034, "theoretical_loss": 5.5617713988168616, "tokens_seen": 36896768 }, { "epoch": 0.01, "learning_rate": 0.000799037111334002, "loss": 3.2984, "theoretical_loss": 5.560219282366614, "tokens_seen": 36962304 }, { "epoch": 0.01, "learning_rate": 0.0007990210631895687, "loss": 3.3318, "theoretical_loss": 5.558670684458994, "tokens_seen": 37027840 }, { "epoch": 0.01, "learning_rate": 0.0007990050150451354, "loss": 3.3465, "theoretical_loss": 5.557125590911339, "tokens_seen": 37093376 }, { "epoch": 0.01, "learning_rate": 0.0007989889669007021, "loss": 3.3319, "theoretical_loss": 5.5555839876230815, "tokens_seen": 37158912 }, { "epoch": 0.01, "learning_rate": 0.0007989729187562688, "loss": 3.2583, "theoretical_loss": 5.55404586057513, "tokens_seen": 37224448 }, { "epoch": 0.01, "learning_rate": 0.0007989568706118355, "loss": 3.1394, "theoretical_loss": 5.5525111958292594, "tokens_seen": 37289984 }, { "epoch": 0.01, "learning_rate": 0.0007989408224674022, "loss": 3.3485, "theoretical_loss": 5.550979979527501, "tokens_seen": 37355520 }, { "epoch": 0.01, "learning_rate": 0.0007989247743229689, "loss": 3.3191, "theoretical_loss": 5.549452197891538, "tokens_seen": 37421056 }, { "epoch": 0.01, "learning_rate": 0.0007989087261785356, "loss": 3.243, "theoretical_loss": 5.547927837222119, "tokens_seen": 37486592 }, { "epoch": 0.01, "learning_rate": 0.0007988926780341023, "loss": 3.128, "theoretical_loss": 5.546406883898451, "tokens_seen": 37552128 }, { "epoch": 0.01, "learning_rate": 0.000798876629889669, "loss": 3.2572, "theoretical_loss": 5.544889324377628, "tokens_seen": 37617664 }, { "epoch": 0.01, "learning_rate": 0.0007988605817452357, "loss": 3.1857, "theoretical_loss": 5.543375145194048, "tokens_seen": 37683200 }, { "epoch": 0.01, "learning_rate": 0.0007988445336008024, "loss": 3.1529, "theoretical_loss": 5.541864332958831, "tokens_seen": 37748736 }, { "epoch": 0.01, "learning_rate": 0.0007988284854563691, "loss": 3.2037, "theoretical_loss": 5.540356874359257, "tokens_seen": 37814272 }, { "epoch": 0.01, "learning_rate": 0.0007988124373119358, "loss": 3.2591, "theoretical_loss": 5.538852756158201, "tokens_seen": 37879808 }, { "epoch": 0.01, "learning_rate": 0.0007987963891675025, "loss": 3.1802, "theoretical_loss": 5.537351965193569, "tokens_seen": 37945344 }, { "epoch": 0.01, "learning_rate": 0.0007987803410230693, "loss": 3.1964, "theoretical_loss": 5.535854488377751, "tokens_seen": 38010880 }, { "epoch": 0.01, "learning_rate": 0.0007987642928786359, "loss": 3.346, "theoretical_loss": 5.534360312697064, "tokens_seen": 38076416 }, { "epoch": 0.01, "learning_rate": 0.0007987482447342027, "loss": 3.1631, "theoretical_loss": 5.532869425211214, "tokens_seen": 38141952 }, { "epoch": 0.01, "learning_rate": 0.0007987321965897693, "loss": 3.2718, "theoretical_loss": 5.531381813052757, "tokens_seen": 38207488 }, { "epoch": 0.01, "learning_rate": 0.0007987161484453361, "loss": 3.17, "theoretical_loss": 5.529897463426559, "tokens_seen": 38273024 }, { "epoch": 0.01, "learning_rate": 0.0007987001003009027, "loss": 3.2378, "theoretical_loss": 5.528416363609276, "tokens_seen": 38338560 }, { "epoch": 0.01, "learning_rate": 0.0007986840521564695, "loss": 3.1696, "theoretical_loss": 5.526938500948817, "tokens_seen": 38404096 }, { "epoch": 0.01, "learning_rate": 0.0007986680040120361, "loss": 3.2103, "theoretical_loss": 5.525463862863836, "tokens_seen": 38469632 }, { "epoch": 0.01, "learning_rate": 0.0007986519558676029, "loss": 3.0849, "theoretical_loss": 5.523992436843209, "tokens_seen": 38535168 }, { "epoch": 0.01, "learning_rate": 0.0007986359077231695, "loss": 3.2743, "theoretical_loss": 5.522524210445528, "tokens_seen": 38600704 }, { "epoch": 0.01, "learning_rate": 0.0007986198595787363, "loss": 3.272, "theoretical_loss": 5.521059171298592, "tokens_seen": 38666240 }, { "epoch": 0.01, "learning_rate": 0.0007986038114343029, "loss": 3.3046, "theoretical_loss": 5.519597307098907, "tokens_seen": 38731776 }, { "epoch": 0.01, "learning_rate": 0.0007985877632898697, "loss": 3.2986, "theoretical_loss": 5.518138605611188, "tokens_seen": 38797312 }, { "epoch": 0.01, "learning_rate": 0.0007985717151454363, "loss": 3.2102, "theoretical_loss": 5.516683054667868, "tokens_seen": 38862848 }, { "epoch": 0.01, "learning_rate": 0.0007985556670010031, "loss": 3.3232, "theoretical_loss": 5.515230642168607, "tokens_seen": 38928384 }, { "epoch": 0.01, "learning_rate": 0.0007985396188565697, "loss": 3.2207, "theoretical_loss": 5.513781356079811, "tokens_seen": 38993920 }, { "epoch": 0.01, "learning_rate": 0.0007985235707121365, "loss": 3.1524, "theoretical_loss": 5.512335184434152, "tokens_seen": 39059456 }, { "epoch": 0.01, "learning_rate": 0.0007985075225677031, "loss": 3.3457, "theoretical_loss": 5.5108921153300905, "tokens_seen": 39124992 }, { "epoch": 0.01, "learning_rate": 0.0007984914744232699, "loss": 3.1989, "theoretical_loss": 5.5094521369314045, "tokens_seen": 39190528 }, { "epoch": 0.01, "learning_rate": 0.0007984754262788365, "loss": 3.0462, "theoretical_loss": 5.5080152374667275, "tokens_seen": 39256064 }, { "epoch": 0.01, "objective/train/docs_used": 19810, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.971580982208252, "objective/train/theoretical_loss": 5.506581405229079, "objective/train/tokens_used": 59781600, "theoretical_loss": 5.506581405229079, "tokens_seen": 39321600 }, { "epoch": 0.01, "learning_rate": 0.0007984593781344033, "loss": 3.0874, "theoretical_loss": 5.506581405229079, "tokens_seen": 39321600 }, { "epoch": 0.01, "learning_rate": 0.0007984433299899699, "loss": 3.2052, "theoretical_loss": 5.505150628575409, "tokens_seen": 39387136 }, { "epoch": 0.01, "learning_rate": 0.0007984272818455367, "loss": 3.164, "theoretical_loss": 5.503722895926143, "tokens_seen": 39452672 }, { "epoch": 0.01, "learning_rate": 0.0007984112337011033, "loss": 3.0422, "theoretical_loss": 5.50229819576473, "tokens_seen": 39518208 }, { "epoch": 0.01, "learning_rate": 0.0007983951855566701, "loss": 3.0343, "theoretical_loss": 5.500876516637199, "tokens_seen": 39583744 }, { "epoch": 0.01, "learning_rate": 0.0007983791374122367, "loss": 3.1188, "theoretical_loss": 5.49945784715171, "tokens_seen": 39649280 }, { "epoch": 0.01, "learning_rate": 0.0007983630892678035, "loss": 3.0288, "theoretical_loss": 5.498042175978123, "tokens_seen": 39714816 }, { "epoch": 0.01, "learning_rate": 0.0007983470411233701, "loss": 3.2065, "theoretical_loss": 5.496629491847553, "tokens_seen": 39780352 }, { "epoch": 0.01, "learning_rate": 0.0007983309929789369, "loss": 3.1265, "theoretical_loss": 5.495219783551947, "tokens_seen": 39845888 }, { "epoch": 0.01, "learning_rate": 0.0007983149448345036, "loss": 3.1179, "theoretical_loss": 5.493813039943654, "tokens_seen": 39911424 }, { "epoch": 0.01, "learning_rate": 0.0007982988966900703, "loss": 2.9897, "theoretical_loss": 5.492409249934996, "tokens_seen": 39976960 }, { "epoch": 0.01, "learning_rate": 0.000798282848545637, "loss": 3.1839, "theoretical_loss": 5.491008402497855, "tokens_seen": 40042496 }, { "epoch": 0.01, "learning_rate": 0.0007982668004012037, "loss": 3.1042, "theoretical_loss": 5.489610486663252, "tokens_seen": 40108032 }, { "epoch": 0.01, "learning_rate": 0.0007982507522567704, "loss": 3.0255, "theoretical_loss": 5.488215491520933, "tokens_seen": 40173568 }, { "epoch": 0.01, "learning_rate": 0.0007982347041123371, "loss": 3.0889, "theoretical_loss": 5.486823406218962, "tokens_seen": 40239104 }, { "epoch": 0.01, "learning_rate": 0.0007982186559679038, "loss": 3.2075, "theoretical_loss": 5.485434219963315, "tokens_seen": 40304640 }, { "epoch": 0.01, "learning_rate": 0.0007982026078234705, "loss": 3.0804, "theoretical_loss": 5.484047922017476, "tokens_seen": 40370176 }, { "epoch": 0.01, "learning_rate": 0.0007981865596790372, "loss": 3.199, "theoretical_loss": 5.482664501702036, "tokens_seen": 40435712 }, { "epoch": 0.01, "learning_rate": 0.0007981705115346039, "loss": 3.0804, "theoretical_loss": 5.481283948394303, "tokens_seen": 40501248 }, { "epoch": 0.01, "learning_rate": 0.0007981544633901706, "loss": 3.1914, "theoretical_loss": 5.479906251527901, "tokens_seen": 40566784 }, { "epoch": 0.01, "learning_rate": 0.0007981384152457373, "loss": 3.1122, "theoretical_loss": 5.478531400592393, "tokens_seen": 40632320 }, { "epoch": 0.01, "learning_rate": 0.000798122367101304, "loss": 3.0153, "theoretical_loss": 5.4771593851328815, "tokens_seen": 40697856 }, { "epoch": 0.01, "learning_rate": 0.0007981063189568707, "loss": 3.0745, "theoretical_loss": 5.475790194749637, "tokens_seen": 40763392 }, { "epoch": 0.01, "learning_rate": 0.0007980902708124374, "loss": 3.2291, "theoretical_loss": 5.474423819097713, "tokens_seen": 40828928 }, { "epoch": 0.01, "learning_rate": 0.0007980742226680041, "loss": 3.072, "theoretical_loss": 5.473060247886572, "tokens_seen": 40894464 }, { "epoch": 0.01, "learning_rate": 0.0007980581745235708, "loss": 3.0078, "theoretical_loss": 5.471699470879715, "tokens_seen": 40960000 }, { "epoch": 0.01, "learning_rate": 0.0007980421263791375, "loss": 3.0983, "theoretical_loss": 5.470341477894303, "tokens_seen": 41025536 }, { "epoch": 0.01, "learning_rate": 0.0007980260782347042, "loss": 3.1138, "theoretical_loss": 5.468986258800805, "tokens_seen": 41091072 }, { "epoch": 0.01, "learning_rate": 0.0007980100300902709, "loss": 3.0204, "theoretical_loss": 5.4676338035226175, "tokens_seen": 41156608 }, { "epoch": 0.01, "learning_rate": 0.0007979939819458376, "loss": 3.2892, "theoretical_loss": 5.466284102035719, "tokens_seen": 41222144 }, { "epoch": 0.01, "learning_rate": 0.0007979779338014043, "loss": 3.1621, "theoretical_loss": 5.464937144368307, "tokens_seen": 41287680 }, { "epoch": 0.01, "learning_rate": 0.000797961885656971, "loss": 3.0173, "theoretical_loss": 5.463592920600435, "tokens_seen": 41353216 }, { "epoch": 0.01, "learning_rate": 0.0007979458375125376, "loss": 3.0774, "theoretical_loss": 5.46225142086368, "tokens_seen": 41418752 }, { "epoch": 0.01, "learning_rate": 0.0007979297893681044, "loss": 3.1238, "theoretical_loss": 5.460912635340775, "tokens_seen": 41484288 }, { "epoch": 0.01, "learning_rate": 0.000797913741223671, "loss": 3.0618, "theoretical_loss": 5.459576554265276, "tokens_seen": 41549824 }, { "epoch": 0.01, "learning_rate": 0.0007978976930792378, "loss": 3.1313, "theoretical_loss": 5.458243167921212, "tokens_seen": 41615360 }, { "epoch": 0.01, "learning_rate": 0.0007978816449348044, "loss": 3.2165, "theoretical_loss": 5.456912466642748, "tokens_seen": 41680896 }, { "epoch": 0.01, "learning_rate": 0.0007978655967903712, "loss": 2.9085, "theoretical_loss": 5.455584440813853, "tokens_seen": 41746432 }, { "epoch": 0.01, "learning_rate": 0.0007978495486459378, "loss": 2.9859, "theoretical_loss": 5.454259080867951, "tokens_seen": 41811968 }, { "epoch": 0.01, "learning_rate": 0.0007978335005015046, "loss": 2.9411, "theoretical_loss": 5.4529363772876085, "tokens_seen": 41877504 }, { "epoch": 0.01, "learning_rate": 0.0007978174523570712, "loss": 3.0094, "theoretical_loss": 5.451616320604193, "tokens_seen": 41943040 }, { "epoch": 0.01, "learning_rate": 0.0007978014042126379, "loss": 2.9767, "theoretical_loss": 5.450298901397547, "tokens_seen": 42008576 }, { "epoch": 0.01, "learning_rate": 0.0007977853560682046, "loss": 3.1042, "theoretical_loss": 5.448984110295672, "tokens_seen": 42074112 }, { "epoch": 0.01, "learning_rate": 0.0007977693079237713, "loss": 3.0762, "theoretical_loss": 5.447671937974404, "tokens_seen": 42139648 }, { "epoch": 0.01, "learning_rate": 0.000797753259779338, "loss": 2.9742, "theoretical_loss": 5.446362375157092, "tokens_seen": 42205184 }, { "epoch": 0.01, "learning_rate": 0.0007977372116349047, "loss": 3.2779, "theoretical_loss": 5.445055412614287, "tokens_seen": 42270720 }, { "epoch": 0.01, "learning_rate": 0.0007977211634904714, "loss": 2.9849, "theoretical_loss": 5.443751041163428, "tokens_seen": 42336256 }, { "epoch": 0.01, "learning_rate": 0.0007977051153460381, "loss": 3.0884, "theoretical_loss": 5.442449251668533, "tokens_seen": 42401792 }, { "epoch": 0.01, "learning_rate": 0.0007976890672016048, "loss": 3.0079, "theoretical_loss": 5.441150035039886, "tokens_seen": 42467328 }, { "epoch": 0.01, "learning_rate": 0.0007976730190571715, "loss": 3.0901, "theoretical_loss": 5.439853382233738, "tokens_seen": 42532864 }, { "epoch": 0.01, "objective/train/docs_used": 20768, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.992009401321411, "objective/train/theoretical_loss": 5.438559284252003, "objective/train/tokens_used": 63058400, "theoretical_loss": 5.438559284252003, "tokens_seen": 42598400 }, { "epoch": 0.01, "learning_rate": 0.0007976569709127382, "loss": 2.9284, "theoretical_loss": 5.438559284252003, "tokens_seen": 42598400 }, { "epoch": 0.01, "learning_rate": 0.0007976409227683049, "loss": 2.9027, "theoretical_loss": 5.437267732141949, "tokens_seen": 42663936 }, { "epoch": 0.01, "learning_rate": 0.0007976248746238716, "loss": 2.9383, "theoretical_loss": 5.435978716995917, "tokens_seen": 42729472 }, { "epoch": 0.01, "learning_rate": 0.0007976088264794383, "loss": 3.0424, "theoretical_loss": 5.434692229951009, "tokens_seen": 42795008 }, { "epoch": 0.01, "learning_rate": 0.000797592778335005, "loss": 3.0061, "theoretical_loss": 5.433408262188802, "tokens_seen": 42860544 }, { "epoch": 0.01, "learning_rate": 0.0007975767301905718, "loss": 3.0298, "theoretical_loss": 5.432126804935059, "tokens_seen": 42926080 }, { "epoch": 0.01, "learning_rate": 0.0007975606820461384, "loss": 3.1409, "theoretical_loss": 5.430847849459438, "tokens_seen": 42991616 }, { "epoch": 0.01, "learning_rate": 0.0007975446339017052, "loss": 3.0722, "theoretical_loss": 5.429571387075203, "tokens_seen": 43057152 }, { "epoch": 0.01, "learning_rate": 0.0007975285857572718, "loss": 3.055, "theoretical_loss": 5.428297409138947, "tokens_seen": 43122688 }, { "epoch": 0.01, "learning_rate": 0.0007975125376128386, "loss": 2.936, "theoretical_loss": 5.427025907050307, "tokens_seen": 43188224 }, { "epoch": 0.01, "learning_rate": 0.0007974964894684052, "loss": 3.0608, "theoretical_loss": 5.425756872251682, "tokens_seen": 43253760 }, { "epoch": 0.01, "learning_rate": 0.000797480441323972, "loss": 2.969, "theoretical_loss": 5.42449029622796, "tokens_seen": 43319296 }, { "epoch": 0.01, "learning_rate": 0.0007974643931795386, "loss": 2.982, "theoretical_loss": 5.423226170506243, "tokens_seen": 43384832 }, { "epoch": 0.01, "learning_rate": 0.0007974483450351054, "loss": 2.959, "theoretical_loss": 5.421964486655567, "tokens_seen": 43450368 }, { "epoch": 0.01, "learning_rate": 0.000797432296890672, "loss": 3.1053, "theoretical_loss": 5.420705236286646, "tokens_seen": 43515904 }, { "epoch": 0.01, "learning_rate": 0.0007974162487462388, "loss": 3.1943, "theoretical_loss": 5.41944841105159, "tokens_seen": 43581440 }, { "epoch": 0.01, "learning_rate": 0.0007974002006018054, "loss": 2.9255, "theoretical_loss": 5.418194002643643, "tokens_seen": 43646976 }, { "epoch": 0.01, "learning_rate": 0.0007973841524573722, "loss": 3.0381, "theoretical_loss": 5.416942002796926, "tokens_seen": 43712512 }, { "epoch": 0.01, "learning_rate": 0.0007973681043129388, "loss": 2.9989, "theoretical_loss": 5.415692403286162, "tokens_seen": 43778048 }, { "epoch": 0.01, "learning_rate": 0.0007973520561685056, "loss": 3.0323, "theoretical_loss": 5.414445195926426, "tokens_seen": 43843584 }, { "epoch": 0.01, "learning_rate": 0.0007973360080240722, "loss": 2.9815, "theoretical_loss": 5.413200372572884, "tokens_seen": 43909120 }, { "epoch": 0.01, "learning_rate": 0.000797319959879639, "loss": 3.0141, "theoretical_loss": 5.411957925120538, "tokens_seen": 43974656 }, { "epoch": 0.01, "learning_rate": 0.0007973039117352056, "loss": 3.0708, "theoretical_loss": 5.410717845503969, "tokens_seen": 44040192 }, { "epoch": 0.01, "learning_rate": 0.0007972878635907724, "loss": 2.8975, "theoretical_loss": 5.40948012569709, "tokens_seen": 44105728 }, { "epoch": 0.01, "learning_rate": 0.000797271815446339, "loss": 3.0037, "theoretical_loss": 5.408244757712893, "tokens_seen": 44171264 }, { "epoch": 0.01, "learning_rate": 0.0007972557673019058, "loss": 2.9777, "theoretical_loss": 5.407011733603199, "tokens_seen": 44236800 }, { "epoch": 0.01, "learning_rate": 0.0007972397191574724, "loss": 2.885, "theoretical_loss": 5.4057810454584185, "tokens_seen": 44302336 }, { "epoch": 0.01, "learning_rate": 0.0007972236710130392, "loss": 2.9414, "theoretical_loss": 5.404552685407301, "tokens_seen": 44367872 }, { "epoch": 0.01, "learning_rate": 0.0007972076228686058, "loss": 3.097, "theoretical_loss": 5.403326645616698, "tokens_seen": 44433408 }, { "epoch": 0.01, "learning_rate": 0.0007971915747241726, "loss": 2.9061, "theoretical_loss": 5.402102918291316, "tokens_seen": 44498944 }, { "epoch": 0.01, "learning_rate": 0.0007971755265797392, "loss": 3.0093, "theoretical_loss": 5.400881495673488, "tokens_seen": 44564480 }, { "epoch": 0.01, "learning_rate": 0.000797159478435306, "loss": 3.103, "theoretical_loss": 5.3996623700429245, "tokens_seen": 44630016 }, { "epoch": 0.01, "learning_rate": 0.0007971434302908726, "loss": 2.9274, "theoretical_loss": 5.398445533716492, "tokens_seen": 44695552 }, { "epoch": 0.01, "learning_rate": 0.0007971273821464394, "loss": 3.0244, "theoretical_loss": 5.3972309790479684, "tokens_seen": 44761088 }, { "epoch": 0.01, "learning_rate": 0.000797111334002006, "loss": 2.9874, "theoretical_loss": 5.39601869842782, "tokens_seen": 44826624 }, { "epoch": 0.01, "learning_rate": 0.0007970952858575728, "loss": 2.893, "theoretical_loss": 5.394808684282968, "tokens_seen": 44892160 }, { "epoch": 0.01, "learning_rate": 0.0007970792377131394, "loss": 2.9119, "theoretical_loss": 5.393600929076564, "tokens_seen": 44957696 }, { "epoch": 0.01, "learning_rate": 0.0007970631895687062, "loss": 2.845, "theoretical_loss": 5.392395425307758, "tokens_seen": 45023232 }, { "epoch": 0.01, "learning_rate": 0.0007970471414242728, "loss": 2.8557, "theoretical_loss": 5.391192165511484, "tokens_seen": 45088768 }, { "epoch": 0.01, "learning_rate": 0.0007970310932798396, "loss": 2.9916, "theoretical_loss": 5.389991142258227, "tokens_seen": 45154304 }, { "epoch": 0.01, "learning_rate": 0.0007970150451354062, "loss": 3.0289, "theoretical_loss": 5.38879234815381, "tokens_seen": 45219840 }, { "epoch": 0.01, "learning_rate": 0.000796998996990973, "loss": 2.857, "theoretical_loss": 5.387595775839175, "tokens_seen": 45285376 }, { "epoch": 0.01, "learning_rate": 0.0007969829488465396, "loss": 2.9597, "theoretical_loss": 5.386401417990159, "tokens_seen": 45350912 }, { "epoch": 0.01, "learning_rate": 0.0007969669007021064, "loss": 2.9621, "theoretical_loss": 5.385209267317286, "tokens_seen": 45416448 }, { "epoch": 0.01, "learning_rate": 0.000796950852557673, "loss": 2.9259, "theoretical_loss": 5.3840193165655466, "tokens_seen": 45481984 }, { "epoch": 0.01, "learning_rate": 0.0007969348044132398, "loss": 2.8076, "theoretical_loss": 5.382831558514191, "tokens_seen": 45547520 }, { "epoch": 0.01, "learning_rate": 0.0007969187562688065, "loss": 2.9423, "theoretical_loss": 5.3816459859765144, "tokens_seen": 45613056 }, { "epoch": 0.01, "learning_rate": 0.0007969027081243732, "loss": 2.7761, "theoretical_loss": 5.380462591799651, "tokens_seen": 45678592 }, { "epoch": 0.01, "learning_rate": 0.0007968866599799399, "loss": 2.8484, "theoretical_loss": 5.379281368864362, "tokens_seen": 45744128 }, { "epoch": 0.01, "learning_rate": 0.0007968706118355066, "loss": 2.8477, "theoretical_loss": 5.378102310084834, "tokens_seen": 45809664 }, { "epoch": 0.01, "objective/train/docs_used": 22096, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.943756580352783, "objective/train/theoretical_loss": 5.376925408408475, "objective/train/tokens_used": 66335200, "theoretical_loss": 5.376925408408475, "tokens_seen": 45875200 }, { "epoch": 0.01, "learning_rate": 0.0007968545636910733, "loss": 2.9656, "theoretical_loss": 5.376925408408475, "tokens_seen": 45875200 }, { "epoch": 0.01, "learning_rate": 0.00079683851554664, "loss": 2.8387, "theoretical_loss": 5.375750656815705, "tokens_seen": 45940736 }, { "epoch": 0.01, "learning_rate": 0.0007968224674022067, "loss": 2.9168, "theoretical_loss": 5.37457804831976, "tokens_seen": 46006272 }, { "epoch": 0.01, "learning_rate": 0.0007968064192577734, "loss": 2.8562, "theoretical_loss": 5.373407575966493, "tokens_seen": 46071808 }, { "epoch": 0.01, "learning_rate": 0.0007967903711133401, "loss": 2.8091, "theoretical_loss": 5.372239232834173, "tokens_seen": 46137344 }, { "epoch": 0.01, "learning_rate": 0.0007967743229689068, "loss": 2.9054, "theoretical_loss": 5.3710730120332855, "tokens_seen": 46202880 }, { "epoch": 0.01, "learning_rate": 0.0007967582748244735, "loss": 2.9022, "theoretical_loss": 5.369908906706346, "tokens_seen": 46268416 }, { "epoch": 0.01, "learning_rate": 0.0007967422266800402, "loss": 2.9656, "theoretical_loss": 5.368746910027696, "tokens_seen": 46333952 }, { "epoch": 0.01, "learning_rate": 0.0007967261785356069, "loss": 2.7845, "theoretical_loss": 5.3675870152033145, "tokens_seen": 46399488 }, { "epoch": 0.01, "learning_rate": 0.0007967101303911736, "loss": 2.8521, "theoretical_loss": 5.36642921547063, "tokens_seen": 46465024 }, { "epoch": 0.01, "learning_rate": 0.0007966940822467403, "loss": 2.8249, "theoretical_loss": 5.365273504098326, "tokens_seen": 46530560 }, { "epoch": 0.01, "learning_rate": 0.000796678034102307, "loss": 2.7694, "theoretical_loss": 5.364119874386157, "tokens_seen": 46596096 }, { "epoch": 0.01, "learning_rate": 0.0007966619859578737, "loss": 2.9288, "theoretical_loss": 5.362968319664756, "tokens_seen": 46661632 }, { "epoch": 0.01, "learning_rate": 0.0007966459378134404, "loss": 2.9025, "theoretical_loss": 5.361818833295455, "tokens_seen": 46727168 }, { "epoch": 0.01, "learning_rate": 0.0007966298896690071, "loss": 2.9219, "theoretical_loss": 5.3606714086701, "tokens_seen": 46792704 }, { "epoch": 0.01, "learning_rate": 0.0007966138415245738, "loss": 2.9099, "theoretical_loss": 5.359526039210861, "tokens_seen": 46858240 }, { "epoch": 0.01, "learning_rate": 0.0007965977933801405, "loss": 2.88, "theoretical_loss": 5.358382718370063, "tokens_seen": 46923776 }, { "epoch": 0.01, "learning_rate": 0.0007965817452357072, "loss": 2.8903, "theoretical_loss": 5.357241439629992, "tokens_seen": 46989312 }, { "epoch": 0.01, "learning_rate": 0.0007965656970912739, "loss": 2.7691, "theoretical_loss": 5.3561021965027305, "tokens_seen": 47054848 }, { "epoch": 0.01, "learning_rate": 0.0007965496489468405, "loss": 2.8294, "theoretical_loss": 5.354964982529967, "tokens_seen": 47120384 }, { "epoch": 0.01, "learning_rate": 0.0007965336008024073, "loss": 2.8102, "theoretical_loss": 5.353829791282831, "tokens_seen": 47185920 }, { "epoch": 0.01, "learning_rate": 0.0007965175526579739, "loss": 2.7822, "theoretical_loss": 5.35269661636171, "tokens_seen": 47251456 }, { "epoch": 0.01, "learning_rate": 0.0007965015045135407, "loss": 2.8876, "theoretical_loss": 5.35156545139608, "tokens_seen": 47316992 }, { "epoch": 0.01, "learning_rate": 0.0007964854563691073, "loss": 2.7635, "theoretical_loss": 5.350436290044332, "tokens_seen": 47382528 }, { "epoch": 0.01, "learning_rate": 0.0007964694082246741, "loss": 2.8377, "theoretical_loss": 5.3493091259936, "tokens_seen": 47448064 }, { "epoch": 0.01, "learning_rate": 0.0007964533600802407, "loss": 2.8324, "theoretical_loss": 5.348183952959596, "tokens_seen": 47513600 }, { "epoch": 0.01, "learning_rate": 0.0007964373119358075, "loss": 2.7908, "theoretical_loss": 5.347060764686436, "tokens_seen": 47579136 }, { "epoch": 0.01, "learning_rate": 0.0007964212637913741, "loss": 2.8496, "theoretical_loss": 5.345939554946472, "tokens_seen": 47644672 }, { "epoch": 0.01, "learning_rate": 0.0007964052156469408, "loss": 2.7488, "theoretical_loss": 5.344820317540132, "tokens_seen": 47710208 }, { "epoch": 0.01, "learning_rate": 0.0007963891675025075, "loss": 2.8114, "theoretical_loss": 5.343703046295751, "tokens_seen": 47775744 }, { "epoch": 0.01, "learning_rate": 0.0007963731193580743, "loss": 2.8671, "theoretical_loss": 5.342587735069408, "tokens_seen": 47841280 }, { "epoch": 0.01, "learning_rate": 0.0007963570712136409, "loss": 2.9205, "theoretical_loss": 5.3414743777447615, "tokens_seen": 47906816 }, { "epoch": 0.01, "learning_rate": 0.0007963410230692077, "loss": 2.7384, "theoretical_loss": 5.340362968232892, "tokens_seen": 47972352 }, { "epoch": 0.01, "learning_rate": 0.0007963249749247743, "loss": 2.9322, "theoretical_loss": 5.339253500472138, "tokens_seen": 48037888 }, { "epoch": 0.01, "learning_rate": 0.0007963089267803411, "loss": 2.6983, "theoretical_loss": 5.338145968427941, "tokens_seen": 48103424 }, { "epoch": 0.01, "learning_rate": 0.0007962928786359077, "loss": 2.8289, "theoretical_loss": 5.337040366092687, "tokens_seen": 48168960 }, { "epoch": 0.01, "learning_rate": 0.0007962768304914745, "loss": 2.9222, "theoretical_loss": 5.335936687485542, "tokens_seen": 48234496 }, { "epoch": 0.01, "learning_rate": 0.0007962607823470411, "loss": 2.7881, "theoretical_loss": 5.334834926652308, "tokens_seen": 48300032 }, { "epoch": 0.01, "learning_rate": 0.0007962447342026079, "loss": 2.7958, "theoretical_loss": 5.33373507766526, "tokens_seen": 48365568 }, { "epoch": 0.01, "learning_rate": 0.0007962286860581745, "loss": 2.8646, "theoretical_loss": 5.332637134622999, "tokens_seen": 48431104 }, { "epoch": 0.01, "learning_rate": 0.0007962126379137413, "loss": 2.7199, "theoretical_loss": 5.331541091650294, "tokens_seen": 48496640 }, { "epoch": 0.01, "learning_rate": 0.0007961965897693079, "loss": 2.8311, "theoretical_loss": 5.330446942897931, "tokens_seen": 48562176 }, { "epoch": 0.01, "learning_rate": 0.0007961805416248747, "loss": 2.8684, "theoretical_loss": 5.32935468254257, "tokens_seen": 48627712 }, { "epoch": 0.01, "learning_rate": 0.0007961644934804413, "loss": 2.822, "theoretical_loss": 5.328264304786586, "tokens_seen": 48693248 }, { "epoch": 0.01, "learning_rate": 0.0007961484453360081, "loss": 2.7969, "theoretical_loss": 5.327175803857929, "tokens_seen": 48758784 }, { "epoch": 0.01, "learning_rate": 0.0007961323971915747, "loss": 2.7726, "theoretical_loss": 5.326089174009972, "tokens_seen": 48824320 }, { "epoch": 0.01, "learning_rate": 0.0007961163490471415, "loss": 2.7235, "theoretical_loss": 5.325004409521368, "tokens_seen": 48889856 }, { "epoch": 0.01, "learning_rate": 0.0007961003009027081, "loss": 2.7985, "theoretical_loss": 5.323921504695903, "tokens_seen": 48955392 }, { "epoch": 0.01, "learning_rate": 0.0007960842527582749, "loss": 2.9327, "theoretical_loss": 5.322840453862356, "tokens_seen": 49020928 }, { "epoch": 0.01, "learning_rate": 0.0007960682046138415, "loss": 2.7259, "theoretical_loss": 5.3217612513743475, "tokens_seen": 49086464 }, { "epoch": 0.01, "objective/train/docs_used": 22959, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.8861923217773438, "objective/train/theoretical_loss": 5.320683891610208, "objective/train/tokens_used": 69612000, "theoretical_loss": 5.320683891610208, "tokens_seen": 49152000 }, { "epoch": 0.01, "learning_rate": 0.0007960521564694083, "loss": 2.8363, "theoretical_loss": 5.320683891610208, "tokens_seen": 49152000 }, { "epoch": 0.01, "learning_rate": 0.0007960361083249749, "loss": 2.8106, "theoretical_loss": 5.3196083689728315, "tokens_seen": 49217536 }, { "epoch": 0.01, "learning_rate": 0.0007960200601805417, "loss": 2.8124, "theoretical_loss": 5.318534677889536, "tokens_seen": 49283072 }, { "epoch": 0.01, "learning_rate": 0.0007960040120361083, "loss": 2.6993, "theoretical_loss": 5.317462812811925, "tokens_seen": 49348608 }, { "epoch": 0.01, "learning_rate": 0.0007959879638916751, "loss": 2.7378, "theoretical_loss": 5.316392768215751, "tokens_seen": 49414144 }, { "epoch": 0.01, "learning_rate": 0.0007959719157472417, "loss": 2.7691, "theoretical_loss": 5.315324538600778, "tokens_seen": 49479680 }, { "epoch": 0.02, "learning_rate": 0.0007959558676028085, "loss": 2.9151, "theoretical_loss": 5.31425811849064, "tokens_seen": 49545216 }, { "epoch": 0.02, "learning_rate": 0.0007959398194583751, "loss": 2.616, "theoretical_loss": 5.313193502432718, "tokens_seen": 49610752 }, { "epoch": 0.02, "learning_rate": 0.0007959237713139419, "loss": 2.7306, "theoretical_loss": 5.312130684997992, "tokens_seen": 49676288 }, { "epoch": 0.02, "learning_rate": 0.0007959077231695085, "loss": 2.801, "theoretical_loss": 5.31106966078092, "tokens_seen": 49741824 }, { "epoch": 0.02, "learning_rate": 0.0007958916750250753, "loss": 2.736, "theoretical_loss": 5.310010424399295, "tokens_seen": 49807360 }, { "epoch": 0.02, "learning_rate": 0.0007958756268806419, "loss": 2.7149, "theoretical_loss": 5.308952970494124, "tokens_seen": 49872896 }, { "epoch": 0.02, "learning_rate": 0.0007958595787362087, "loss": 2.7643, "theoretical_loss": 5.307897293729489, "tokens_seen": 49938432 }, { "epoch": 0.02, "learning_rate": 0.0007958435305917753, "loss": 2.7688, "theoretical_loss": 5.306843388792423, "tokens_seen": 50003968 }, { "epoch": 0.02, "learning_rate": 0.0007958274824473421, "loss": 2.6344, "theoretical_loss": 5.305791250392778, "tokens_seen": 50069504 }, { "epoch": 0.02, "learning_rate": 0.0007958114343029087, "loss": 2.7253, "theoretical_loss": 5.304740873263098, "tokens_seen": 50135040 }, { "epoch": 0.02, "learning_rate": 0.0007957953861584755, "loss": 2.7176, "theoretical_loss": 5.303692252158496, "tokens_seen": 50200576 }, { "epoch": 0.02, "learning_rate": 0.0007957793380140421, "loss": 2.7815, "theoretical_loss": 5.302645381856522, "tokens_seen": 50266112 }, { "epoch": 0.02, "learning_rate": 0.0007957632898696089, "loss": 2.7198, "theoretical_loss": 5.30160025715704, "tokens_seen": 50331648 }, { "epoch": 0.02, "learning_rate": 0.0007957472417251755, "loss": 2.5469, "theoretical_loss": 5.300556872882108, "tokens_seen": 50397184 }, { "epoch": 0.02, "learning_rate": 0.0007957311935807423, "loss": 2.6499, "theoretical_loss": 5.299515223875848, "tokens_seen": 50462720 }, { "epoch": 0.02, "learning_rate": 0.0007957151454363089, "loss": 2.7165, "theoretical_loss": 5.2984753050043265, "tokens_seen": 50528256 }, { "epoch": 0.02, "learning_rate": 0.0007956990972918757, "loss": 2.6517, "theoretical_loss": 5.2974371111554355, "tokens_seen": 50593792 }, { "epoch": 0.02, "learning_rate": 0.0007956830491474423, "loss": 2.6449, "theoretical_loss": 5.296400637238764, "tokens_seen": 50659328 }, { "epoch": 0.02, "learning_rate": 0.0007956670010030091, "loss": 2.5609, "theoretical_loss": 5.295365878185488, "tokens_seen": 50724864 }, { "epoch": 0.02, "learning_rate": 0.0007956509528585757, "loss": 2.4886, "theoretical_loss": 5.294332828948243, "tokens_seen": 50790400 }, { "epoch": 0.02, "learning_rate": 0.0007956349047141425, "loss": 2.6212, "theoretical_loss": 5.2933014845010105, "tokens_seen": 50855936 }, { "epoch": 0.02, "learning_rate": 0.0007956188565697091, "loss": 2.6007, "theoretical_loss": 5.292271839838996, "tokens_seen": 50921472 }, { "epoch": 0.02, "learning_rate": 0.0007956028084252759, "loss": 2.7466, "theoretical_loss": 5.291243889978516, "tokens_seen": 50987008 }, { "epoch": 0.02, "learning_rate": 0.0007955867602808426, "loss": 2.7354, "theoretical_loss": 5.290217629956882, "tokens_seen": 51052544 }, { "epoch": 0.02, "learning_rate": 0.0007955707121364093, "loss": 2.8293, "theoretical_loss": 5.289193054832282, "tokens_seen": 51118080 }, { "epoch": 0.02, "learning_rate": 0.000795554663991976, "loss": 2.7158, "theoretical_loss": 5.288170159683668, "tokens_seen": 51183616 }, { "epoch": 0.02, "learning_rate": 0.0007955386158475427, "loss": 2.6579, "theoretical_loss": 5.287148939610645, "tokens_seen": 51249152 }, { "epoch": 0.02, "learning_rate": 0.0007955225677031094, "loss": 2.5181, "theoretical_loss": 5.28612938973335, "tokens_seen": 51314688 }, { "epoch": 0.02, "learning_rate": 0.0007955065195586761, "loss": 2.7695, "theoretical_loss": 5.285111505192349, "tokens_seen": 51380224 }, { "epoch": 0.02, "learning_rate": 0.0007954904714142428, "loss": 2.6868, "theoretical_loss": 5.284095281148522, "tokens_seen": 51445760 }, { "epoch": 0.02, "learning_rate": 0.0007954744232698095, "loss": 2.747, "theoretical_loss": 5.283080712782951, "tokens_seen": 51511296 }, { "epoch": 0.02, "learning_rate": 0.0007954583751253762, "loss": 2.6446, "theoretical_loss": 5.282067795296811, "tokens_seen": 51576832 }, { "epoch": 0.02, "learning_rate": 0.0007954423269809429, "loss": 2.6414, "theoretical_loss": 5.28105652391126, "tokens_seen": 51642368 }, { "epoch": 0.02, "learning_rate": 0.0007954262788365096, "loss": 2.6543, "theoretical_loss": 5.280046893867333, "tokens_seen": 51707904 }, { "epoch": 0.02, "learning_rate": 0.0007954102306920763, "loss": 2.5674, "theoretical_loss": 5.279038900425832, "tokens_seen": 51773440 }, { "epoch": 0.02, "learning_rate": 0.000795394182547643, "loss": 2.6305, "theoretical_loss": 5.278032538867221, "tokens_seen": 51838976 }, { "epoch": 0.02, "learning_rate": 0.0007953781344032097, "loss": 2.5573, "theoretical_loss": 5.277027804491517, "tokens_seen": 51904512 }, { "epoch": 0.02, "learning_rate": 0.0007953620862587764, "loss": 2.6, "theoretical_loss": 5.2760246926181855, "tokens_seen": 51970048 }, { "epoch": 0.02, "learning_rate": 0.0007953460381143431, "loss": 2.6884, "theoretical_loss": 5.275023198586037, "tokens_seen": 52035584 }, { "epoch": 0.02, "learning_rate": 0.0007953299899699098, "loss": 2.6203, "theoretical_loss": 5.274023317753125, "tokens_seen": 52101120 }, { "epoch": 0.02, "learning_rate": 0.0007953139418254765, "loss": 2.6784, "theoretical_loss": 5.273025045496631, "tokens_seen": 52166656 }, { "epoch": 0.02, "learning_rate": 0.0007952978936810432, "loss": 2.5281, "theoretical_loss": 5.272028377212777, "tokens_seen": 52232192 }, { "epoch": 0.02, "learning_rate": 0.0007952818455366099, "loss": 2.5807, "theoretical_loss": 5.271033308316715, "tokens_seen": 52297728 }, { "epoch": 0.02, "learning_rate": 0.0007952657973921766, "loss": 2.6348, "theoretical_loss": 5.270039834242425, "tokens_seen": 52363264 }, { "epoch": 0.02, "objective/train/docs_used": 24243, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.5289902687072754, "objective/train/theoretical_loss": 5.269047950442619, "objective/train/tokens_used": 72888800, "theoretical_loss": 5.269047950442619, "tokens_seen": 52428800 }, { "epoch": 0.02, "learning_rate": 0.0007952497492477433, "loss": 2.485, "theoretical_loss": 5.269047950442619, "tokens_seen": 52428800 }, { "epoch": 0.02, "learning_rate": 0.00079523370110331, "loss": 2.6376, "theoretical_loss": 5.2680576523886335, "tokens_seen": 52494336 }, { "epoch": 0.02, "learning_rate": 0.0007952176529588767, "loss": 2.7194, "theoretical_loss": 5.2670689355703395, "tokens_seen": 52559872 }, { "epoch": 0.02, "learning_rate": 0.0007952016048144434, "loss": 2.6341, "theoretical_loss": 5.266081795496035, "tokens_seen": 52625408 }, { "epoch": 0.02, "learning_rate": 0.0007951855566700101, "loss": 2.5662, "theoretical_loss": 5.265096227692354, "tokens_seen": 52690944 }, { "epoch": 0.02, "learning_rate": 0.0007951695085255768, "loss": 2.6624, "theoretical_loss": 5.264112227704162, "tokens_seen": 52756480 }, { "epoch": 0.02, "learning_rate": 0.0007951534603811434, "loss": 2.7557, "theoretical_loss": 5.263129791094466, "tokens_seen": 52822016 }, { "epoch": 0.02, "learning_rate": 0.0007951374122367102, "loss": 2.6062, "theoretical_loss": 5.26214891344431, "tokens_seen": 52887552 }, { "epoch": 0.02, "learning_rate": 0.0007951213640922768, "loss": 2.5699, "theoretical_loss": 5.26116959035269, "tokens_seen": 52953088 }, { "epoch": 0.02, "learning_rate": 0.0007951053159478436, "loss": 2.506, "theoretical_loss": 5.260191817436452, "tokens_seen": 53018624 }, { "epoch": 0.02, "learning_rate": 0.0007950892678034102, "loss": 2.6012, "theoretical_loss": 5.259215590330195, "tokens_seen": 53084160 }, { "epoch": 0.02, "learning_rate": 0.000795073219658977, "loss": 2.5899, "theoretical_loss": 5.258240904686186, "tokens_seen": 53149696 }, { "epoch": 0.02, "learning_rate": 0.0007950571715145436, "loss": 2.5064, "theoretical_loss": 5.25726775617426, "tokens_seen": 53215232 }, { "epoch": 0.02, "learning_rate": 0.0007950411233701104, "loss": 2.4435, "theoretical_loss": 5.256296140481728, "tokens_seen": 53280768 }, { "epoch": 0.02, "learning_rate": 0.000795025075225677, "loss": 2.5262, "theoretical_loss": 5.255326053313292, "tokens_seen": 53346304 }, { "epoch": 0.02, "learning_rate": 0.0007950090270812438, "loss": 2.5646, "theoretical_loss": 5.254357490390941, "tokens_seen": 53411840 }, { "epoch": 0.02, "learning_rate": 0.0007949929789368104, "loss": 2.4133, "theoretical_loss": 5.253390447453873, "tokens_seen": 53477376 }, { "epoch": 0.02, "learning_rate": 0.0007949769307923772, "loss": 2.6187, "theoretical_loss": 5.252424920258396, "tokens_seen": 53542912 }, { "epoch": 0.02, "learning_rate": 0.0007949608826479438, "loss": 2.5912, "theoretical_loss": 5.251460904577845, "tokens_seen": 53608448 }, { "epoch": 0.02, "learning_rate": 0.0007949448345035106, "loss": 2.5508, "theoretical_loss": 5.250498396202486, "tokens_seen": 53673984 }, { "epoch": 0.02, "learning_rate": 0.0007949287863590772, "loss": 2.5112, "theoretical_loss": 5.249537390939436, "tokens_seen": 53739520 }, { "epoch": 0.02, "learning_rate": 0.000794912738214644, "loss": 2.4452, "theoretical_loss": 5.248577884612564, "tokens_seen": 53805056 }, { "epoch": 0.02, "learning_rate": 0.0007948966900702106, "loss": 2.4712, "theoretical_loss": 5.247619873062416, "tokens_seen": 53870592 }, { "epoch": 0.02, "learning_rate": 0.0007948806419257774, "loss": 2.5985, "theoretical_loss": 5.246663352146122, "tokens_seen": 53936128 }, { "epoch": 0.02, "learning_rate": 0.000794864593781344, "loss": 2.6442, "theoretical_loss": 5.245708317737305, "tokens_seen": 54001664 }, { "epoch": 0.02, "learning_rate": 0.0007948485456369108, "loss": 2.4876, "theoretical_loss": 5.244754765726007, "tokens_seen": 54067200 }, { "epoch": 0.02, "learning_rate": 0.0007948324974924774, "loss": 2.5376, "theoretical_loss": 5.243802692018596, "tokens_seen": 54132736 }, { "epoch": 0.02, "learning_rate": 0.0007948164493480442, "loss": 2.4553, "theoretical_loss": 5.242852092537677, "tokens_seen": 54198272 }, { "epoch": 0.02, "learning_rate": 0.0007948004012036108, "loss": 2.6929, "theoretical_loss": 5.241902963222023, "tokens_seen": 54263808 }, { "epoch": 0.02, "learning_rate": 0.0007947843530591776, "loss": 2.5035, "theoretical_loss": 5.240955300026476, "tokens_seen": 54329344 }, { "epoch": 0.02, "learning_rate": 0.0007947683049147442, "loss": 2.5704, "theoretical_loss": 5.240009098921874, "tokens_seen": 54394880 }, { "epoch": 0.02, "learning_rate": 0.000794752256770311, "loss": 2.4209, "theoretical_loss": 5.239064355894965, "tokens_seen": 54460416 }, { "epoch": 0.02, "learning_rate": 0.0007947362086258776, "loss": 2.5877, "theoretical_loss": 5.238121066948324, "tokens_seen": 54525952 }, { "epoch": 0.02, "learning_rate": 0.0007947201604814444, "loss": 2.4356, "theoretical_loss": 5.237179228100272, "tokens_seen": 54591488 }, { "epoch": 0.02, "learning_rate": 0.000794704112337011, "loss": 2.498, "theoretical_loss": 5.2362388353848, "tokens_seen": 54657024 }, { "epoch": 0.02, "learning_rate": 0.0007946880641925778, "loss": 2.426, "theoretical_loss": 5.23529988485148, "tokens_seen": 54722560 }, { "epoch": 0.02, "learning_rate": 0.0007946720160481444, "loss": 2.527, "theoretical_loss": 5.2343623725653945, "tokens_seen": 54788096 }, { "epoch": 0.02, "learning_rate": 0.0007946559679037112, "loss": 2.5915, "theoretical_loss": 5.233426294607049, "tokens_seen": 54853632 }, { "epoch": 0.02, "learning_rate": 0.0007946399197592778, "loss": 2.5706, "theoretical_loss": 5.232491647072299, "tokens_seen": 54919168 }, { "epoch": 0.02, "learning_rate": 0.0007946238716148446, "loss": 2.5656, "theoretical_loss": 5.231558426072269, "tokens_seen": 54984704 }, { "epoch": 0.02, "learning_rate": 0.0007946078234704112, "loss": 2.4501, "theoretical_loss": 5.230626627733274, "tokens_seen": 55050240 }, { "epoch": 0.02, "learning_rate": 0.000794591775325978, "loss": 2.5487, "theoretical_loss": 5.2296962481967455, "tokens_seen": 55115776 }, { "epoch": 0.02, "learning_rate": 0.0007945757271815446, "loss": 2.3679, "theoretical_loss": 5.228767283619151, "tokens_seen": 55181312 }, { "epoch": 0.02, "learning_rate": 0.0007945596790371114, "loss": 2.6326, "theoretical_loss": 5.227839730171918, "tokens_seen": 55246848 }, { "epoch": 0.02, "learning_rate": 0.000794543630892678, "loss": 2.4476, "theoretical_loss": 5.2269135840413625, "tokens_seen": 55312384 }, { "epoch": 0.02, "learning_rate": 0.0007945275827482448, "loss": 2.309, "theoretical_loss": 5.225988841428606, "tokens_seen": 55377920 }, { "epoch": 0.02, "learning_rate": 0.0007945115346038114, "loss": 2.2936, "theoretical_loss": 5.225065498549508, "tokens_seen": 55443456 }, { "epoch": 0.02, "learning_rate": 0.0007944954864593782, "loss": 2.4656, "theoretical_loss": 5.224143551634588, "tokens_seen": 55508992 }, { "epoch": 0.02, "learning_rate": 0.0007944794383149448, "loss": 2.428, "theoretical_loss": 5.22322299692895, "tokens_seen": 55574528 }, { "epoch": 0.02, "learning_rate": 0.0007944633901705116, "loss": 2.4442, "theoretical_loss": 5.222303830692211, "tokens_seen": 55640064 }, { "epoch": 0.02, "objective/train/docs_used": 25141, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.6088738441467285, "objective/train/theoretical_loss": 5.22138604919843, "objective/train/tokens_used": 76165600, "theoretical_loss": 5.22138604919843, "tokens_seen": 55705600 }, { "epoch": 0.02, "learning_rate": 0.0007944473420260782, "loss": 2.5252, "theoretical_loss": 5.22138604919843, "tokens_seen": 55705600 }, { "epoch": 0.02, "learning_rate": 0.000794431293881645, "loss": 2.6352, "theoretical_loss": 5.220469648736031, "tokens_seen": 55771136 }, { "epoch": 0.02, "learning_rate": 0.0007944152457372116, "loss": 2.4382, "theoretical_loss": 5.219554625607731, "tokens_seen": 55836672 }, { "epoch": 0.02, "learning_rate": 0.0007943991975927784, "loss": 2.5073, "theoretical_loss": 5.218640976130474, "tokens_seen": 55902208 }, { "epoch": 0.02, "learning_rate": 0.000794383149448345, "loss": 2.502, "theoretical_loss": 5.217728696635353, "tokens_seen": 55967744 }, { "epoch": 0.02, "learning_rate": 0.0007943671013039118, "loss": 2.3945, "theoretical_loss": 5.216817783467543, "tokens_seen": 56033280 }, { "epoch": 0.02, "learning_rate": 0.0007943510531594784, "loss": 2.6394, "theoretical_loss": 5.215908232986227, "tokens_seen": 56098816 }, { "epoch": 0.02, "learning_rate": 0.0007943350050150452, "loss": 2.4114, "theoretical_loss": 5.215000041564533, "tokens_seen": 56164352 }, { "epoch": 0.02, "learning_rate": 0.0007943189568706118, "loss": 2.47, "theoretical_loss": 5.214093205589455, "tokens_seen": 56229888 }, { "epoch": 0.02, "learning_rate": 0.0007943029087261786, "loss": 2.4864, "theoretical_loss": 5.213187721461791, "tokens_seen": 56295424 }, { "epoch": 0.02, "learning_rate": 0.0007942868605817452, "loss": 2.5074, "theoretical_loss": 5.212283585596071, "tokens_seen": 56360960 }, { "epoch": 0.02, "learning_rate": 0.000794270812437312, "loss": 2.3196, "theoretical_loss": 5.211380794420492, "tokens_seen": 56426496 }, { "epoch": 0.02, "learning_rate": 0.0007942547642928786, "loss": 2.3877, "theoretical_loss": 5.210479344376845, "tokens_seen": 56492032 }, { "epoch": 0.02, "learning_rate": 0.0007942387161484454, "loss": 2.5349, "theoretical_loss": 5.209579231920451, "tokens_seen": 56557568 }, { "epoch": 0.02, "learning_rate": 0.000794222668004012, "loss": 2.4561, "theoretical_loss": 5.208680453520092, "tokens_seen": 56623104 }, { "epoch": 0.02, "learning_rate": 0.0007942066198595788, "loss": 2.4743, "theoretical_loss": 5.2077830056579515, "tokens_seen": 56688640 }, { "epoch": 0.02, "learning_rate": 0.0007941905717151455, "loss": 2.5124, "theoretical_loss": 5.206886884829535, "tokens_seen": 56754176 }, { "epoch": 0.02, "learning_rate": 0.0007941745235707122, "loss": 2.3724, "theoretical_loss": 5.205992087543614, "tokens_seen": 56819712 }, { "epoch": 0.02, "learning_rate": 0.0007941584754262789, "loss": 2.5328, "theoretical_loss": 5.205098610322162, "tokens_seen": 56885248 }, { "epoch": 0.02, "learning_rate": 0.0007941424272818456, "loss": 2.6109, "theoretical_loss": 5.204206449700279, "tokens_seen": 56950784 }, { "epoch": 0.02, "learning_rate": 0.0007941263791374123, "loss": 2.4425, "theoretical_loss": 5.203315602226139, "tokens_seen": 57016320 }, { "epoch": 0.02, "learning_rate": 0.000794110330992979, "loss": 2.353, "theoretical_loss": 5.2024260644609175, "tokens_seen": 57081856 }, { "epoch": 0.02, "learning_rate": 0.0007940942828485457, "loss": 2.4324, "theoretical_loss": 5.201537832978731, "tokens_seen": 57147392 }, { "epoch": 0.02, "learning_rate": 0.0007940782347041124, "loss": 2.3614, "theoretical_loss": 5.200650904366574, "tokens_seen": 57212928 }, { "epoch": 0.02, "learning_rate": 0.0007940621865596791, "loss": 2.4856, "theoretical_loss": 5.199765275224252, "tokens_seen": 57278464 }, { "epoch": 0.02, "learning_rate": 0.0007940461384152458, "loss": 2.4181, "theoretical_loss": 5.198880942164323, "tokens_seen": 57344000 }, { "epoch": 0.02, "learning_rate": 0.0007940300902708125, "loss": 2.351, "theoretical_loss": 5.197997901812036, "tokens_seen": 57409536 }, { "epoch": 0.02, "learning_rate": 0.0007940140421263792, "loss": 2.5241, "theoretical_loss": 5.197116150805264, "tokens_seen": 57475072 }, { "epoch": 0.02, "learning_rate": 0.0007939979939819459, "loss": 2.3707, "theoretical_loss": 5.196235685794446, "tokens_seen": 57540608 }, { "epoch": 0.02, "learning_rate": 0.0007939819458375126, "loss": 2.4242, "theoretical_loss": 5.195356503442525, "tokens_seen": 57606144 }, { "epoch": 0.02, "learning_rate": 0.0007939658976930794, "loss": 2.4264, "theoretical_loss": 5.194478600424889, "tokens_seen": 57671680 }, { "epoch": 0.02, "learning_rate": 0.000793949849548646, "loss": 2.4905, "theoretical_loss": 5.193601973429306, "tokens_seen": 57737216 }, { "epoch": 0.02, "learning_rate": 0.0007939338014042128, "loss": 2.3264, "theoretical_loss": 5.192726619155867, "tokens_seen": 57802752 }, { "epoch": 0.02, "learning_rate": 0.0007939177532597794, "loss": 2.2947, "theoretical_loss": 5.191852534316928, "tokens_seen": 57868288 }, { "epoch": 0.02, "learning_rate": 0.0007939017051153462, "loss": 2.5367, "theoretical_loss": 5.190979715637049, "tokens_seen": 57933824 }, { "epoch": 0.02, "learning_rate": 0.0007938856569709128, "loss": 2.4345, "theoretical_loss": 5.190108159852929, "tokens_seen": 57999360 }, { "epoch": 0.02, "learning_rate": 0.0007938696088264795, "loss": 2.48, "theoretical_loss": 5.189237863713357, "tokens_seen": 58064896 }, { "epoch": 0.02, "learning_rate": 0.0007938535606820462, "loss": 2.5036, "theoretical_loss": 5.188368823979152, "tokens_seen": 58130432 }, { "epoch": 0.02, "learning_rate": 0.0007938375125376129, "loss": 2.2739, "theoretical_loss": 5.187501037423096, "tokens_seen": 58195968 }, { "epoch": 0.02, "learning_rate": 0.0007938214643931796, "loss": 2.3866, "theoretical_loss": 5.186634500829886, "tokens_seen": 58261504 }, { "epoch": 0.02, "learning_rate": 0.0007938054162487463, "loss": 2.4664, "theoretical_loss": 5.185769210996076, "tokens_seen": 58327040 }, { "epoch": 0.02, "learning_rate": 0.000793789368104313, "loss": 2.2739, "theoretical_loss": 5.184905164730012, "tokens_seen": 58392576 }, { "epoch": 0.02, "learning_rate": 0.0007937733199598797, "loss": 2.4244, "theoretical_loss": 5.184042358851785, "tokens_seen": 58458112 }, { "epoch": 0.02, "learning_rate": 0.0007937572718154463, "loss": 2.3718, "theoretical_loss": 5.1831807901931715, "tokens_seen": 58523648 }, { "epoch": 0.02, "learning_rate": 0.0007937412236710131, "loss": 2.2701, "theoretical_loss": 5.18232045559757, "tokens_seen": 58589184 }, { "epoch": 0.02, "learning_rate": 0.0007937251755265797, "loss": 2.3716, "theoretical_loss": 5.181461351919959, "tokens_seen": 58654720 }, { "epoch": 0.02, "learning_rate": 0.0007937091273821465, "loss": 2.5091, "theoretical_loss": 5.180603476026832, "tokens_seen": 58720256 }, { "epoch": 0.02, "learning_rate": 0.0007936930792377131, "loss": 2.3163, "theoretical_loss": 5.179746824796146, "tokens_seen": 58785792 }, { "epoch": 0.02, "learning_rate": 0.0007936770310932799, "loss": 2.2918, "theoretical_loss": 5.178891395117262, "tokens_seen": 58851328 }, { "epoch": 0.02, "learning_rate": 0.0007936609829488465, "loss": 2.4097, "theoretical_loss": 5.178037183890902, "tokens_seen": 58916864 }, { "epoch": 0.02, "objective/train/docs_used": 26367, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.4226865768432617, "objective/train/theoretical_loss": 5.177184188029082, "objective/train/tokens_used": 79442400, "theoretical_loss": 5.177184188029082, "tokens_seen": 58982400 }, { "epoch": 0.02, "learning_rate": 0.0007936449348044133, "loss": 2.3572, "theoretical_loss": 5.177184188029082, "tokens_seen": 58982400 }, { "epoch": 0.02, "learning_rate": 0.0007936288866599799, "loss": 2.452, "theoretical_loss": 5.1763324044550645, "tokens_seen": 59047936 }, { "epoch": 0.02, "learning_rate": 0.0007936128385155467, "loss": 2.3822, "theoretical_loss": 5.175481830103307, "tokens_seen": 59113472 }, { "epoch": 0.02, "learning_rate": 0.0007935967903711133, "loss": 2.376, "theoretical_loss": 5.174632461919406, "tokens_seen": 59179008 }, { "epoch": 0.02, "learning_rate": 0.0007935807422266801, "loss": 2.3096, "theoretical_loss": 5.173784296860043, "tokens_seen": 59244544 }, { "epoch": 0.02, "learning_rate": 0.0007935646940822467, "loss": 2.4012, "theoretical_loss": 5.172937331892934, "tokens_seen": 59310080 }, { "epoch": 0.02, "learning_rate": 0.0007935486459378135, "loss": 2.2779, "theoretical_loss": 5.17209156399678, "tokens_seen": 59375616 }, { "epoch": 0.02, "learning_rate": 0.0007935325977933801, "loss": 2.4276, "theoretical_loss": 5.171246990161208, "tokens_seen": 59441152 }, { "epoch": 0.02, "learning_rate": 0.0007935165496489469, "loss": 2.4266, "theoretical_loss": 5.17040360738673, "tokens_seen": 59506688 }, { "epoch": 0.02, "learning_rate": 0.0007935005015045135, "loss": 2.2993, "theoretical_loss": 5.169561412684677, "tokens_seen": 59572224 }, { "epoch": 0.02, "learning_rate": 0.0007934844533600803, "loss": 2.4694, "theoretical_loss": 5.168720403077165, "tokens_seen": 59637760 }, { "epoch": 0.02, "learning_rate": 0.0007934684052156469, "loss": 2.2188, "theoretical_loss": 5.167880575597031, "tokens_seen": 59703296 }, { "epoch": 0.02, "learning_rate": 0.0007934523570712137, "loss": 2.2279, "theoretical_loss": 5.167041927287787, "tokens_seen": 59768832 }, { "epoch": 0.02, "learning_rate": 0.0007934363089267803, "loss": 2.4385, "theoretical_loss": 5.166204455203575, "tokens_seen": 59834368 }, { "epoch": 0.02, "learning_rate": 0.0007934202607823471, "loss": 2.2816, "theoretical_loss": 5.16536815640911, "tokens_seen": 59899904 }, { "epoch": 0.02, "learning_rate": 0.0007934042126379137, "loss": 2.4746, "theoretical_loss": 5.1645330279796315, "tokens_seen": 59965440 }, { "epoch": 0.02, "learning_rate": 0.0007933881644934805, "loss": 2.3473, "theoretical_loss": 5.1636990670008585, "tokens_seen": 60030976 }, { "epoch": 0.02, "learning_rate": 0.0007933721163490471, "loss": 2.4463, "theoretical_loss": 5.162866270568938, "tokens_seen": 60096512 }, { "epoch": 0.02, "learning_rate": 0.0007933560682046139, "loss": 2.3849, "theoretical_loss": 5.162034635790397, "tokens_seen": 60162048 }, { "epoch": 0.02, "learning_rate": 0.0007933400200601805, "loss": 2.384, "theoretical_loss": 5.1612041597820895, "tokens_seen": 60227584 }, { "epoch": 0.02, "learning_rate": 0.0007933239719157473, "loss": 2.4535, "theoretical_loss": 5.160374839671159, "tokens_seen": 60293120 }, { "epoch": 0.02, "learning_rate": 0.0007933079237713139, "loss": 2.3911, "theoretical_loss": 5.1595466725949795, "tokens_seen": 60358656 }, { "epoch": 0.02, "learning_rate": 0.0007932918756268807, "loss": 2.3502, "theoretical_loss": 5.158719655701115, "tokens_seen": 60424192 }, { "epoch": 0.02, "learning_rate": 0.0007932758274824473, "loss": 2.4282, "theoretical_loss": 5.157893786147269, "tokens_seen": 60489728 }, { "epoch": 0.02, "learning_rate": 0.0007932597793380141, "loss": 2.3445, "theoretical_loss": 5.157069061101238, "tokens_seen": 60555264 }, { "epoch": 0.02, "learning_rate": 0.0007932437311935807, "loss": 2.3545, "theoretical_loss": 5.156245477740866, "tokens_seen": 60620800 }, { "epoch": 0.02, "learning_rate": 0.0007932276830491475, "loss": 2.2794, "theoretical_loss": 5.155423033253997, "tokens_seen": 60686336 }, { "epoch": 0.02, "learning_rate": 0.0007932116349047141, "loss": 2.2316, "theoretical_loss": 5.154601724838431, "tokens_seen": 60751872 }, { "epoch": 0.02, "learning_rate": 0.0007931955867602809, "loss": 2.3536, "theoretical_loss": 5.153781549701872, "tokens_seen": 60817408 }, { "epoch": 0.02, "learning_rate": 0.0007931795386158475, "loss": 2.4524, "theoretical_loss": 5.15296250506189, "tokens_seen": 60882944 }, { "epoch": 0.02, "learning_rate": 0.0007931634904714143, "loss": 2.3219, "theoretical_loss": 5.152144588145871, "tokens_seen": 60948480 }, { "epoch": 0.02, "learning_rate": 0.0007931474423269809, "loss": 2.2977, "theoretical_loss": 5.151327796190975, "tokens_seen": 61014016 }, { "epoch": 0.02, "learning_rate": 0.0007931313941825477, "loss": 2.2728, "theoretical_loss": 5.1505121264440845, "tokens_seen": 61079552 }, { "epoch": 0.02, "learning_rate": 0.0007931153460381143, "loss": 2.4291, "theoretical_loss": 5.149697576161769, "tokens_seen": 61145088 }, { "epoch": 0.02, "learning_rate": 0.0007930992978936811, "loss": 2.258, "theoretical_loss": 5.148884142610232, "tokens_seen": 61210624 }, { "epoch": 0.02, "learning_rate": 0.0007930832497492477, "loss": 2.3553, "theoretical_loss": 5.148071823065274, "tokens_seen": 61276160 }, { "epoch": 0.02, "learning_rate": 0.0007930672016048145, "loss": 2.3682, "theoretical_loss": 5.147260614812246, "tokens_seen": 61341696 }, { "epoch": 0.02, "learning_rate": 0.0007930511534603811, "loss": 2.3699, "theoretical_loss": 5.146450515146002, "tokens_seen": 61407232 }, { "epoch": 0.02, "learning_rate": 0.0007930351053159479, "loss": 2.2222, "theoretical_loss": 5.145641521370861, "tokens_seen": 61472768 }, { "epoch": 0.02, "learning_rate": 0.0007930190571715145, "loss": 2.3013, "theoretical_loss": 5.144833630800562, "tokens_seen": 61538304 }, { "epoch": 0.02, "learning_rate": 0.0007930030090270813, "loss": 2.3248, "theoretical_loss": 5.1440268407582215, "tokens_seen": 61603840 }, { "epoch": 0.02, "learning_rate": 0.0007929869608826479, "loss": 2.3713, "theoretical_loss": 5.143221148576288, "tokens_seen": 61669376 }, { "epoch": 0.02, "learning_rate": 0.0007929709127382147, "loss": 2.1181, "theoretical_loss": 5.142416551596505, "tokens_seen": 61734912 }, { "epoch": 0.02, "learning_rate": 0.0007929548645937813, "loss": 2.1712, "theoretical_loss": 5.141613047169862, "tokens_seen": 61800448 }, { "epoch": 0.02, "learning_rate": 0.0007929388164493481, "loss": 2.3425, "theoretical_loss": 5.14081063265656, "tokens_seen": 61865984 }, { "epoch": 0.02, "learning_rate": 0.0007929227683049147, "loss": 2.2708, "theoretical_loss": 5.140009305425963, "tokens_seen": 61931520 }, { "epoch": 0.02, "learning_rate": 0.0007929067201604815, "loss": 2.205, "theoretical_loss": 5.13920906285656, "tokens_seen": 61997056 }, { "epoch": 0.02, "learning_rate": 0.0007928906720160481, "loss": 2.3632, "theoretical_loss": 5.138409902335925, "tokens_seen": 62062592 }, { "epoch": 0.02, "learning_rate": 0.0007928746238716149, "loss": 2.2693, "theoretical_loss": 5.137611821260673, "tokens_seen": 62128128 }, { "epoch": 0.02, "learning_rate": 0.0007928585757271815, "loss": 2.2251, "theoretical_loss": 5.13681481703642, "tokens_seen": 62193664 }, { "epoch": 0.02, "objective/train/docs_used": 27626, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.3975930213928223, "objective/train/theoretical_loss": 5.136018887077743, "objective/train/tokens_used": 82719200, "theoretical_loss": 5.136018887077743, "tokens_seen": 62259200 }, { "epoch": 0.02, "learning_rate": 0.0007928425275827483, "loss": 2.3101, "theoretical_loss": 5.136018887077743, "tokens_seen": 62259200 }, { "epoch": 0.02, "learning_rate": 0.000792826479438315, "loss": 2.446, "theoretical_loss": 5.13522402880814, "tokens_seen": 62324736 }, { "epoch": 0.02, "learning_rate": 0.0007928104312938817, "loss": 2.4389, "theoretical_loss": 5.13443023965999, "tokens_seen": 62390272 }, { "epoch": 0.02, "learning_rate": 0.0007927943831494484, "loss": 2.3297, "theoretical_loss": 5.13363751707451, "tokens_seen": 62455808 }, { "epoch": 0.02, "learning_rate": 0.0007927783350050151, "loss": 2.2738, "theoretical_loss": 5.132845858501721, "tokens_seen": 62521344 }, { "epoch": 0.02, "learning_rate": 0.0007927622868605819, "loss": 2.2318, "theoretical_loss": 5.132055261400403, "tokens_seen": 62586880 }, { "epoch": 0.02, "learning_rate": 0.0007927462387161485, "loss": 2.193, "theoretical_loss": 5.131265723238059, "tokens_seen": 62652416 }, { "epoch": 0.02, "learning_rate": 0.0007927301905717153, "loss": 2.2944, "theoretical_loss": 5.130477241490876, "tokens_seen": 62717952 }, { "epoch": 0.02, "learning_rate": 0.0007927141424272819, "loss": 2.2733, "theoretical_loss": 5.129689813643683, "tokens_seen": 62783488 }, { "epoch": 0.02, "learning_rate": 0.0007926980942828487, "loss": 2.2248, "theoretical_loss": 5.128903437189917, "tokens_seen": 62849024 }, { "epoch": 0.02, "learning_rate": 0.0007926820461384153, "loss": 2.2887, "theoretical_loss": 5.128118109631581, "tokens_seen": 62914560 }, { "epoch": 0.02, "learning_rate": 0.0007926659979939821, "loss": 2.31, "theoretical_loss": 5.127333828479208, "tokens_seen": 62980096 }, { "epoch": 0.02, "learning_rate": 0.0007926499498495487, "loss": 2.2063, "theoretical_loss": 5.126550591251821, "tokens_seen": 63045632 }, { "epoch": 0.02, "learning_rate": 0.0007926339017051155, "loss": 2.3662, "theoretical_loss": 5.125768395476896, "tokens_seen": 63111168 }, { "epoch": 0.02, "learning_rate": 0.0007926178535606821, "loss": 2.2167, "theoretical_loss": 5.124987238690329, "tokens_seen": 63176704 }, { "epoch": 0.02, "learning_rate": 0.0007926018054162489, "loss": 2.3044, "theoretical_loss": 5.124207118436388, "tokens_seen": 63242240 }, { "epoch": 0.02, "learning_rate": 0.0007925857572718155, "loss": 2.3506, "theoretical_loss": 5.123428032267688, "tokens_seen": 63307776 }, { "epoch": 0.02, "learning_rate": 0.0007925697091273823, "loss": 2.2806, "theoretical_loss": 5.122649977745146, "tokens_seen": 63373312 }, { "epoch": 0.02, "learning_rate": 0.0007925536609829489, "loss": 2.3118, "theoretical_loss": 5.121872952437947, "tokens_seen": 63438848 }, { "epoch": 0.02, "learning_rate": 0.0007925376128385157, "loss": 2.3615, "theoretical_loss": 5.121096953923507, "tokens_seen": 63504384 }, { "epoch": 0.02, "learning_rate": 0.0007925215646940823, "loss": 2.3037, "theoretical_loss": 5.1203219797874375, "tokens_seen": 63569920 }, { "epoch": 0.02, "learning_rate": 0.0007925055165496491, "loss": 2.3262, "theoretical_loss": 5.119548027623509, "tokens_seen": 63635456 }, { "epoch": 0.02, "learning_rate": 0.0007924894684052157, "loss": 2.3319, "theoretical_loss": 5.118775095033612, "tokens_seen": 63700992 }, { "epoch": 0.02, "learning_rate": 0.0007924734202607824, "loss": 2.2909, "theoretical_loss": 5.118003179627729, "tokens_seen": 63766528 }, { "epoch": 0.02, "learning_rate": 0.0007924573721163491, "loss": 2.24, "theoretical_loss": 5.11723227902389, "tokens_seen": 63832064 }, { "epoch": 0.02, "learning_rate": 0.0007924413239719158, "loss": 2.3249, "theoretical_loss": 5.1164623908481435, "tokens_seen": 63897600 }, { "epoch": 0.02, "learning_rate": 0.0007924252758274825, "loss": 2.2101, "theoretical_loss": 5.115693512734515, "tokens_seen": 63963136 }, { "epoch": 0.02, "learning_rate": 0.0007924092276830492, "loss": 2.1202, "theoretical_loss": 5.114925642324982, "tokens_seen": 64028672 }, { "epoch": 0.02, "learning_rate": 0.0007923931795386159, "loss": 2.267, "theoretical_loss": 5.114158777269429, "tokens_seen": 64094208 }, { "epoch": 0.02, "learning_rate": 0.0007923771313941826, "loss": 2.2321, "theoretical_loss": 5.113392915225617, "tokens_seen": 64159744 }, { "epoch": 0.02, "learning_rate": 0.0007923610832497493, "loss": 2.35, "theoretical_loss": 5.112628053859153, "tokens_seen": 64225280 }, { "epoch": 0.02, "learning_rate": 0.000792345035105316, "loss": 2.0774, "theoretical_loss": 5.1118641908434475, "tokens_seen": 64290816 }, { "epoch": 0.02, "learning_rate": 0.0007923289869608826, "loss": 2.2412, "theoretical_loss": 5.111101323859687, "tokens_seen": 64356352 }, { "epoch": 0.02, "learning_rate": 0.0007923129388164494, "loss": 2.3277, "theoretical_loss": 5.1103394505968005, "tokens_seen": 64421888 }, { "epoch": 0.02, "learning_rate": 0.000792296890672016, "loss": 2.315, "theoretical_loss": 5.109578568751417, "tokens_seen": 64487424 }, { "epoch": 0.02, "learning_rate": 0.0007922808425275828, "loss": 2.413, "theoretical_loss": 5.108818676027845, "tokens_seen": 64552960 }, { "epoch": 0.02, "learning_rate": 0.0007922647943831494, "loss": 2.1771, "theoretical_loss": 5.108059770138027, "tokens_seen": 64618496 }, { "epoch": 0.02, "learning_rate": 0.0007922487462387162, "loss": 2.3026, "theoretical_loss": 5.107301848801519, "tokens_seen": 64684032 }, { "epoch": 0.02, "learning_rate": 0.0007922326980942828, "loss": 2.253, "theoretical_loss": 5.106544909745443, "tokens_seen": 64749568 }, { "epoch": 0.02, "learning_rate": 0.0007922166499498496, "loss": 2.1783, "theoretical_loss": 5.105788950704467, "tokens_seen": 64815104 }, { "epoch": 0.02, "learning_rate": 0.0007922006018054162, "loss": 2.2324, "theoretical_loss": 5.105033969420764, "tokens_seen": 64880640 }, { "epoch": 0.02, "learning_rate": 0.000792184553660983, "loss": 2.2459, "theoretical_loss": 5.104279963643982, "tokens_seen": 64946176 }, { "epoch": 0.02, "learning_rate": 0.0007921685055165496, "loss": 2.1426, "theoretical_loss": 5.103526931131217, "tokens_seen": 65011712 }, { "epoch": 0.02, "learning_rate": 0.0007921524573721164, "loss": 2.2505, "theoretical_loss": 5.102774869646972, "tokens_seen": 65077248 }, { "epoch": 0.02, "learning_rate": 0.000792136409227683, "loss": 2.0972, "theoretical_loss": 5.102023776963131, "tokens_seen": 65142784 }, { "epoch": 0.02, "learning_rate": 0.0007921203610832498, "loss": 2.1683, "theoretical_loss": 5.101273650858926, "tokens_seen": 65208320 }, { "epoch": 0.02, "learning_rate": 0.0007921043129388164, "loss": 2.2065, "theoretical_loss": 5.100524489120902, "tokens_seen": 65273856 }, { "epoch": 0.02, "learning_rate": 0.0007920882647943832, "loss": 2.2527, "theoretical_loss": 5.099776289542893, "tokens_seen": 65339392 }, { "epoch": 0.02, "learning_rate": 0.0007920722166499498, "loss": 2.1725, "theoretical_loss": 5.099029049925985, "tokens_seen": 65404928 }, { "epoch": 0.02, "learning_rate": 0.0007920561685055166, "loss": 2.2228, "theoretical_loss": 5.098282768078483, "tokens_seen": 65470464 }, { "epoch": 0.02, "objective/train/docs_used": 28581, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.2630820274353027, "objective/train/theoretical_loss": 5.097537441815887, "objective/train/tokens_used": 85996000, "theoretical_loss": 5.097537441815887, "tokens_seen": 65536000 }, { "epoch": 0.02, "learning_rate": 0.0007920401203610832, "loss": 2.244, "theoretical_loss": 5.097537441815887, "tokens_seen": 65536000 }, { "epoch": 0.02, "learning_rate": 0.00079202407221665, "loss": 2.2649, "theoretical_loss": 5.096793068960857, "tokens_seen": 65601536 }, { "epoch": 0.02, "learning_rate": 0.0007920080240722166, "loss": 2.2591, "theoretical_loss": 5.096049647343182, "tokens_seen": 65667072 }, { "epoch": 0.02, "learning_rate": 0.0007919919759277834, "loss": 2.256, "theoretical_loss": 5.095307174799752, "tokens_seen": 65732608 }, { "epoch": 0.02, "learning_rate": 0.00079197592778335, "loss": 2.2502, "theoretical_loss": 5.094565649174524, "tokens_seen": 65798144 }, { "epoch": 0.02, "learning_rate": 0.0007919598796389168, "loss": 2.1983, "theoretical_loss": 5.0938250683184965, "tokens_seen": 65863680 }, { "epoch": 0.02, "learning_rate": 0.0007919438314944834, "loss": 2.261, "theoretical_loss": 5.093085430089674, "tokens_seen": 65929216 }, { "epoch": 0.02, "learning_rate": 0.0007919277833500502, "loss": 2.3129, "theoretical_loss": 5.092346732353046, "tokens_seen": 65994752 }, { "epoch": 0.02, "learning_rate": 0.0007919117352056168, "loss": 2.2523, "theoretical_loss": 5.091608972980545, "tokens_seen": 66060288 }, { "epoch": 0.02, "learning_rate": 0.0007918956870611836, "loss": 2.2646, "theoretical_loss": 5.090872149851029, "tokens_seen": 66125824 }, { "epoch": 0.02, "learning_rate": 0.0007918796389167502, "loss": 2.2916, "theoretical_loss": 5.090136260850245, "tokens_seen": 66191360 }, { "epoch": 0.02, "learning_rate": 0.000791863590772317, "loss": 2.1602, "theoretical_loss": 5.089401303870801, "tokens_seen": 66256896 }, { "epoch": 0.02, "learning_rate": 0.0007918475426278836, "loss": 2.2429, "theoretical_loss": 5.088667276812138, "tokens_seen": 66322432 }, { "epoch": 0.02, "learning_rate": 0.0007918314944834504, "loss": 2.1725, "theoretical_loss": 5.087934177580502, "tokens_seen": 66387968 }, { "epoch": 0.02, "learning_rate": 0.000791815446339017, "loss": 2.2774, "theoretical_loss": 5.087202004088914, "tokens_seen": 66453504 }, { "epoch": 0.02, "learning_rate": 0.0007917993981945838, "loss": 2.1667, "theoretical_loss": 5.086470754257139, "tokens_seen": 66519040 }, { "epoch": 0.02, "learning_rate": 0.0007917833500501504, "loss": 2.2032, "theoretical_loss": 5.0857404260116645, "tokens_seen": 66584576 }, { "epoch": 0.02, "learning_rate": 0.0007917673019057172, "loss": 2.148, "theoretical_loss": 5.085011017285662, "tokens_seen": 66650112 }, { "epoch": 0.02, "learning_rate": 0.0007917512537612838, "loss": 2.2979, "theoretical_loss": 5.084282526018972, "tokens_seen": 66715648 }, { "epoch": 0.02, "learning_rate": 0.0007917352056168506, "loss": 2.2236, "theoretical_loss": 5.083554950158062, "tokens_seen": 66781184 }, { "epoch": 0.02, "learning_rate": 0.0007917191574724172, "loss": 2.1649, "theoretical_loss": 5.0828282876560085, "tokens_seen": 66846720 }, { "epoch": 0.02, "learning_rate": 0.000791703109327984, "loss": 2.2443, "theoretical_loss": 5.0821025364724655, "tokens_seen": 66912256 }, { "epoch": 0.02, "learning_rate": 0.0007916870611835506, "loss": 2.3334, "theoretical_loss": 5.081377694573638, "tokens_seen": 66977792 }, { "epoch": 0.02, "learning_rate": 0.0007916710130391174, "loss": 2.2181, "theoretical_loss": 5.080653759932255, "tokens_seen": 67043328 }, { "epoch": 0.02, "learning_rate": 0.000791654964894684, "loss": 2.1852, "theoretical_loss": 5.079930730527538, "tokens_seen": 67108864 }, { "epoch": 0.02, "learning_rate": 0.0007916389167502508, "loss": 2.1556, "theoretical_loss": 5.079208604345181, "tokens_seen": 67174400 }, { "epoch": 0.02, "learning_rate": 0.0007916228686058174, "loss": 2.2345, "theoretical_loss": 5.078487379377318, "tokens_seen": 67239936 }, { "epoch": 0.02, "learning_rate": 0.0007916068204613842, "loss": 2.1317, "theoretical_loss": 5.0777670536225, "tokens_seen": 67305472 }, { "epoch": 0.02, "learning_rate": 0.000791590772316951, "loss": 2.1145, "theoretical_loss": 5.07704762508566, "tokens_seen": 67371008 }, { "epoch": 0.02, "learning_rate": 0.0007915747241725176, "loss": 2.1057, "theoretical_loss": 5.076329091778101, "tokens_seen": 67436544 }, { "epoch": 0.02, "learning_rate": 0.0007915586760280844, "loss": 2.1766, "theoretical_loss": 5.075611451717457, "tokens_seen": 67502080 }, { "epoch": 0.02, "learning_rate": 0.000791542627883651, "loss": 2.1238, "theoretical_loss": 5.0748947029276685, "tokens_seen": 67567616 }, { "epoch": 0.02, "learning_rate": 0.0007915265797392178, "loss": 2.2196, "theoretical_loss": 5.074178843438963, "tokens_seen": 67633152 }, { "epoch": 0.02, "learning_rate": 0.0007915105315947844, "loss": 2.2516, "theoretical_loss": 5.073463871287823, "tokens_seen": 67698688 }, { "epoch": 0.02, "learning_rate": 0.0007914944834503512, "loss": 2.2305, "theoretical_loss": 5.07274978451696, "tokens_seen": 67764224 }, { "epoch": 0.02, "learning_rate": 0.0007914784353059178, "loss": 2.2993, "theoretical_loss": 5.072036581175293, "tokens_seen": 67829760 }, { "epoch": 0.02, "learning_rate": 0.0007914623871614846, "loss": 2.2447, "theoretical_loss": 5.071324259317921, "tokens_seen": 67895296 }, { "epoch": 0.02, "learning_rate": 0.0007914463390170512, "loss": 2.1236, "theoretical_loss": 5.070612817006094, "tokens_seen": 67960832 }, { "epoch": 0.02, "learning_rate": 0.000791430290872618, "loss": 2.3058, "theoretical_loss": 5.06990225230719, "tokens_seen": 68026368 }, { "epoch": 0.02, "learning_rate": 0.0007914142427281846, "loss": 2.1259, "theoretical_loss": 5.069192563294694, "tokens_seen": 68091904 }, { "epoch": 0.02, "learning_rate": 0.0007913981945837514, "loss": 2.3627, "theoretical_loss": 5.068483748048166, "tokens_seen": 68157440 }, { "epoch": 0.02, "learning_rate": 0.000791382146439318, "loss": 2.3496, "theoretical_loss": 5.06777580465322, "tokens_seen": 68222976 }, { "epoch": 0.02, "learning_rate": 0.0007913660982948848, "loss": 2.315, "theoretical_loss": 5.067068731201498, "tokens_seen": 68288512 }, { "epoch": 0.02, "learning_rate": 0.0007913500501504514, "loss": 2.1493, "theoretical_loss": 5.066362525790646, "tokens_seen": 68354048 }, { "epoch": 0.02, "learning_rate": 0.0007913340020060182, "loss": 2.113, "theoretical_loss": 5.065657186524287, "tokens_seen": 68419584 }, { "epoch": 0.02, "learning_rate": 0.0007913179538615848, "loss": 2.2397, "theoretical_loss": 5.064952711512001, "tokens_seen": 68485120 }, { "epoch": 0.02, "learning_rate": 0.0007913019057171516, "loss": 2.2762, "theoretical_loss": 5.064249098869295, "tokens_seen": 68550656 }, { "epoch": 0.02, "learning_rate": 0.0007912858575727182, "loss": 2.2793, "theoretical_loss": 5.063546346717585, "tokens_seen": 68616192 }, { "epoch": 0.02, "learning_rate": 0.000791269809428285, "loss": 2.2232, "theoretical_loss": 5.062844453184166, "tokens_seen": 68681728 }, { "epoch": 0.02, "learning_rate": 0.0007912537612838516, "loss": 2.2233, "theoretical_loss": 5.062143416402192, "tokens_seen": 68747264 }, { "epoch": 0.02, "objective/train/docs_used": 29892, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.302198648452759, "objective/train/theoretical_loss": 5.061443234510648, "objective/train/tokens_used": 89272800, "theoretical_loss": 5.061443234510648, "tokens_seen": 68812800 }, { "epoch": 0.02, "learning_rate": 0.0007912377131394184, "loss": 2.2377, "theoretical_loss": 5.061443234510648, "tokens_seen": 68812800 }, { "epoch": 0.02, "learning_rate": 0.000791221664994985, "loss": 2.2414, "theoretical_loss": 5.060743905654332, "tokens_seen": 68878336 }, { "epoch": 0.02, "learning_rate": 0.0007912056168505518, "loss": 2.2892, "theoretical_loss": 5.0600454279838285, "tokens_seen": 68943872 }, { "epoch": 0.02, "learning_rate": 0.0007911895687061184, "loss": 2.1666, "theoretical_loss": 5.059347799655483, "tokens_seen": 69009408 }, { "epoch": 0.02, "learning_rate": 0.0007911735205616852, "loss": 2.1416, "theoretical_loss": 5.05865101883138, "tokens_seen": 69074944 }, { "epoch": 0.02, "learning_rate": 0.0007911574724172518, "loss": 2.1452, "theoretical_loss": 5.057955083679321, "tokens_seen": 69140480 }, { "epoch": 0.02, "learning_rate": 0.0007911414242728186, "loss": 2.2087, "theoretical_loss": 5.0572599923728, "tokens_seen": 69206016 }, { "epoch": 0.02, "learning_rate": 0.0007911253761283852, "loss": 2.2024, "theoretical_loss": 5.056565743090982, "tokens_seen": 69271552 }, { "epoch": 0.02, "learning_rate": 0.000791109327983952, "loss": 2.0612, "theoretical_loss": 5.055872334018677, "tokens_seen": 69337088 }, { "epoch": 0.02, "learning_rate": 0.0007910932798395186, "loss": 2.2113, "theoretical_loss": 5.0551797633463185, "tokens_seen": 69402624 }, { "epoch": 0.02, "learning_rate": 0.0007910772316950853, "loss": 2.1887, "theoretical_loss": 5.054488029269946, "tokens_seen": 69468160 }, { "epoch": 0.02, "learning_rate": 0.000791061183550652, "loss": 2.2918, "theoretical_loss": 5.053797129991171, "tokens_seen": 69533696 }, { "epoch": 0.02, "learning_rate": 0.0007910451354062187, "loss": 2.2494, "theoretical_loss": 5.053107063717167, "tokens_seen": 69599232 }, { "epoch": 0.02, "learning_rate": 0.0007910290872617854, "loss": 2.2314, "theoretical_loss": 5.052417828660639, "tokens_seen": 69664768 }, { "epoch": 0.02, "learning_rate": 0.0007910130391173521, "loss": 2.2445, "theoretical_loss": 5.051729423039802, "tokens_seen": 69730304 }, { "epoch": 0.02, "learning_rate": 0.0007909969909729188, "loss": 2.2076, "theoretical_loss": 5.0510418450783625, "tokens_seen": 69795840 }, { "epoch": 0.02, "learning_rate": 0.0007909809428284855, "loss": 2.2421, "theoretical_loss": 5.050355093005495, "tokens_seen": 69861376 }, { "epoch": 0.02, "learning_rate": 0.0007909648946840522, "loss": 2.0975, "theoretical_loss": 5.049669165055816, "tokens_seen": 69926912 }, { "epoch": 0.02, "learning_rate": 0.0007909488465396189, "loss": 2.1393, "theoretical_loss": 5.048984059469369, "tokens_seen": 69992448 }, { "epoch": 0.02, "learning_rate": 0.0007909327983951855, "loss": 2.1042, "theoretical_loss": 5.048299774491598, "tokens_seen": 70057984 }, { "epoch": 0.02, "learning_rate": 0.0007909167502507523, "loss": 2.1535, "theoretical_loss": 5.047616308373328, "tokens_seen": 70123520 }, { "epoch": 0.02, "learning_rate": 0.0007909007021063189, "loss": 2.2246, "theoretical_loss": 5.046933659370742, "tokens_seen": 70189056 }, { "epoch": 0.02, "learning_rate": 0.0007908846539618857, "loss": 2.1902, "theoretical_loss": 5.046251825745361, "tokens_seen": 70254592 }, { "epoch": 0.02, "learning_rate": 0.0007908686058174523, "loss": 2.247, "theoretical_loss": 5.0455708057640205, "tokens_seen": 70320128 }, { "epoch": 0.02, "learning_rate": 0.0007908525576730191, "loss": 2.0948, "theoretical_loss": 5.044890597698853, "tokens_seen": 70385664 }, { "epoch": 0.02, "learning_rate": 0.0007908365095285857, "loss": 2.2894, "theoretical_loss": 5.044211199827263, "tokens_seen": 70451200 }, { "epoch": 0.02, "learning_rate": 0.0007908204613841525, "loss": 2.1606, "theoretical_loss": 5.04353261043191, "tokens_seen": 70516736 }, { "epoch": 0.02, "learning_rate": 0.0007908044132397191, "loss": 2.2046, "theoretical_loss": 5.0428548278006815, "tokens_seen": 70582272 }, { "epoch": 0.02, "learning_rate": 0.0007907883650952859, "loss": 2.0641, "theoretical_loss": 5.042177850226682, "tokens_seen": 70647808 }, { "epoch": 0.02, "learning_rate": 0.0007907723169508525, "loss": 2.1669, "theoretical_loss": 5.0415016760082, "tokens_seen": 70713344 }, { "epoch": 0.02, "learning_rate": 0.0007907562688064193, "loss": 2.1651, "theoretical_loss": 5.040826303448698, "tokens_seen": 70778880 }, { "epoch": 0.02, "learning_rate": 0.0007907402206619859, "loss": 2.1432, "theoretical_loss": 5.040151730856788, "tokens_seen": 70844416 }, { "epoch": 0.02, "learning_rate": 0.0007907241725175527, "loss": 2.1211, "theoretical_loss": 5.039477956546207, "tokens_seen": 70909952 }, { "epoch": 0.02, "learning_rate": 0.0007907081243731193, "loss": 2.2094, "theoretical_loss": 5.038804978835808, "tokens_seen": 70975488 }, { "epoch": 0.02, "learning_rate": 0.0007906920762286861, "loss": 2.2419, "theoretical_loss": 5.038132796049523, "tokens_seen": 71041024 }, { "epoch": 0.02, "learning_rate": 0.0007906760280842527, "loss": 2.1796, "theoretical_loss": 5.037461406516362, "tokens_seen": 71106560 }, { "epoch": 0.02, "learning_rate": 0.0007906599799398195, "loss": 2.1457, "theoretical_loss": 5.036790808570377, "tokens_seen": 71172096 }, { "epoch": 0.02, "learning_rate": 0.0007906439317953861, "loss": 2.1222, "theoretical_loss": 5.036121000550651, "tokens_seen": 71237632 }, { "epoch": 0.02, "learning_rate": 0.0007906278836509529, "loss": 2.2514, "theoretical_loss": 5.035451980801276, "tokens_seen": 71303168 }, { "epoch": 0.02, "learning_rate": 0.0007906118355065195, "loss": 2.1864, "theoretical_loss": 5.034783747671332, "tokens_seen": 71368704 }, { "epoch": 0.02, "learning_rate": 0.0007905957873620863, "loss": 2.148, "theoretical_loss": 5.034116299514871, "tokens_seen": 71434240 }, { "epoch": 0.02, "learning_rate": 0.0007905797392176529, "loss": 2.1176, "theoretical_loss": 5.033449634690893, "tokens_seen": 71499776 }, { "epoch": 0.02, "learning_rate": 0.0007905636910732197, "loss": 2.0967, "theoretical_loss": 5.032783751563327, "tokens_seen": 71565312 }, { "epoch": 0.02, "learning_rate": 0.0007905476429287863, "loss": 2.1888, "theoretical_loss": 5.032118648501017, "tokens_seen": 71630848 }, { "epoch": 0.02, "learning_rate": 0.0007905315947843531, "loss": 2.3203, "theoretical_loss": 5.031454323877698, "tokens_seen": 71696384 }, { "epoch": 0.02, "learning_rate": 0.0007905155466399197, "loss": 2.1714, "theoretical_loss": 5.030790776071976, "tokens_seen": 71761920 }, { "epoch": 0.02, "learning_rate": 0.0007904994984954865, "loss": 2.0794, "theoretical_loss": 5.030128003467312, "tokens_seen": 71827456 }, { "epoch": 0.02, "learning_rate": 0.0007904834503510531, "loss": 2.1015, "theoretical_loss": 5.029466004452004, "tokens_seen": 71892992 }, { "epoch": 0.02, "learning_rate": 0.0007904674022066199, "loss": 2.0396, "theoretical_loss": 5.028804777419165, "tokens_seen": 71958528 }, { "epoch": 0.02, "learning_rate": 0.0007904513540621865, "loss": 2.1914, "theoretical_loss": 5.028144320766701, "tokens_seen": 72024064 }, { "epoch": 0.02, "objective/train/docs_used": 30813, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.0049593448638916, "objective/train/theoretical_loss": 5.0274846328973055, "objective/train/tokens_used": 92549600, "theoretical_loss": 5.0274846328973055, "tokens_seen": 72089600 }, { "epoch": 0.02, "learning_rate": 0.0007904353059177533, "loss": 2.1007, "theoretical_loss": 5.0274846328973055, "tokens_seen": 72089600 }, { "epoch": 0.02, "learning_rate": 0.0007904192577733199, "loss": 2.2275, "theoretical_loss": 5.0268257122184234, "tokens_seen": 72155136 }, { "epoch": 0.02, "learning_rate": 0.0007904032096288867, "loss": 2.1329, "theoretical_loss": 5.026167557142247, "tokens_seen": 72220672 }, { "epoch": 0.02, "learning_rate": 0.0007903871614844534, "loss": 2.098, "theoretical_loss": 5.02551016608569, "tokens_seen": 72286208 }, { "epoch": 0.02, "learning_rate": 0.0007903711133400201, "loss": 2.2127, "theoretical_loss": 5.024853537470369, "tokens_seen": 72351744 }, { "epoch": 0.02, "learning_rate": 0.0007903550651955868, "loss": 2.0479, "theoretical_loss": 5.02419766972259, "tokens_seen": 72417280 }, { "epoch": 0.02, "learning_rate": 0.0007903390170511535, "loss": 2.2583, "theoretical_loss": 5.023542561273327, "tokens_seen": 72482816 }, { "epoch": 0.02, "learning_rate": 0.0007903229689067202, "loss": 2.2263, "theoretical_loss": 5.022888210558206, "tokens_seen": 72548352 }, { "epoch": 0.02, "learning_rate": 0.0007903069207622869, "loss": 2.2046, "theoretical_loss": 5.02223461601748, "tokens_seen": 72613888 }, { "epoch": 0.02, "learning_rate": 0.0007902908726178537, "loss": 2.3318, "theoretical_loss": 5.021581776096022, "tokens_seen": 72679424 }, { "epoch": 0.02, "learning_rate": 0.0007902748244734203, "loss": 2.131, "theoretical_loss": 5.0209296892433, "tokens_seen": 72744960 }, { "epoch": 0.02, "learning_rate": 0.000790258776328987, "loss": 2.2132, "theoretical_loss": 5.020278353913362, "tokens_seen": 72810496 }, { "epoch": 0.02, "learning_rate": 0.0007902427281845537, "loss": 2.0379, "theoretical_loss": 5.019627768564816, "tokens_seen": 72876032 }, { "epoch": 0.02, "learning_rate": 0.0007902266800401205, "loss": 2.1585, "theoretical_loss": 5.018977931660814, "tokens_seen": 72941568 }, { "epoch": 0.02, "learning_rate": 0.0007902106318956871, "loss": 2.1721, "theoretical_loss": 5.018328841669037, "tokens_seen": 73007104 }, { "epoch": 0.02, "learning_rate": 0.0007901945837512539, "loss": 2.3544, "theoretical_loss": 5.017680497061671, "tokens_seen": 73072640 }, { "epoch": 0.02, "learning_rate": 0.0007901785356068205, "loss": 2.175, "theoretical_loss": 5.017032896315397, "tokens_seen": 73138176 }, { "epoch": 0.02, "learning_rate": 0.0007901624874623873, "loss": 2.0713, "theoretical_loss": 5.016386037911371, "tokens_seen": 73203712 }, { "epoch": 0.02, "learning_rate": 0.0007901464393179539, "loss": 2.2291, "theoretical_loss": 5.015739920335203, "tokens_seen": 73269248 }, { "epoch": 0.02, "learning_rate": 0.0007901303911735207, "loss": 2.185, "theoretical_loss": 5.015094542076945, "tokens_seen": 73334784 }, { "epoch": 0.02, "learning_rate": 0.0007901143430290873, "loss": 2.09, "theoretical_loss": 5.014449901631075, "tokens_seen": 73400320 }, { "epoch": 0.02, "learning_rate": 0.0007900982948846541, "loss": 2.1841, "theoretical_loss": 5.013805997496473, "tokens_seen": 73465856 }, { "epoch": 0.02, "learning_rate": 0.0007900822467402207, "loss": 2.1307, "theoretical_loss": 5.013162828176412, "tokens_seen": 73531392 }, { "epoch": 0.02, "learning_rate": 0.0007900661985957875, "loss": 2.2505, "theoretical_loss": 5.012520392178539, "tokens_seen": 73596928 }, { "epoch": 0.02, "learning_rate": 0.0007900501504513541, "loss": 2.1488, "theoretical_loss": 5.011878688014852, "tokens_seen": 73662464 }, { "epoch": 0.02, "learning_rate": 0.0007900341023069209, "loss": 2.1883, "theoretical_loss": 5.011237714201694, "tokens_seen": 73728000 }, { "epoch": 0.02, "learning_rate": 0.0007900180541624875, "loss": 2.0638, "theoretical_loss": 5.010597469259729, "tokens_seen": 73793536 }, { "epoch": 0.02, "learning_rate": 0.0007900020060180543, "loss": 2.1461, "theoretical_loss": 5.009957951713927, "tokens_seen": 73859072 }, { "epoch": 0.02, "learning_rate": 0.0007899859578736209, "loss": 2.1418, "theoretical_loss": 5.009319160093551, "tokens_seen": 73924608 }, { "epoch": 0.02, "learning_rate": 0.0007899699097291877, "loss": 2.1104, "theoretical_loss": 5.008681092932137, "tokens_seen": 73990144 }, { "epoch": 0.02, "learning_rate": 0.0007899538615847543, "loss": 2.2886, "theoretical_loss": 5.008043748767477, "tokens_seen": 74055680 }, { "epoch": 0.02, "learning_rate": 0.0007899378134403211, "loss": 2.1436, "theoretical_loss": 5.007407126141608, "tokens_seen": 74121216 }, { "epoch": 0.02, "learning_rate": 0.0007899217652958877, "loss": 2.171, "theoretical_loss": 5.00677122360079, "tokens_seen": 74186752 }, { "epoch": 0.02, "learning_rate": 0.0007899057171514545, "loss": 2.189, "theoretical_loss": 5.006136039695496, "tokens_seen": 74252288 }, { "epoch": 0.02, "learning_rate": 0.0007898896690070211, "loss": 2.1575, "theoretical_loss": 5.005501572980391, "tokens_seen": 74317824 }, { "epoch": 0.02, "learning_rate": 0.0007898736208625879, "loss": 2.0729, "theoretical_loss": 5.004867822014317, "tokens_seen": 74383360 }, { "epoch": 0.02, "learning_rate": 0.0007898575727181545, "loss": 2.144, "theoretical_loss": 5.004234785360282, "tokens_seen": 74448896 }, { "epoch": 0.02, "learning_rate": 0.0007898415245737213, "loss": 2.1052, "theoretical_loss": 5.0036024615854355, "tokens_seen": 74514432 }, { "epoch": 0.02, "learning_rate": 0.0007898254764292879, "loss": 2.113, "theoretical_loss": 5.002970849261065, "tokens_seen": 74579968 }, { "epoch": 0.02, "learning_rate": 0.0007898094282848547, "loss": 2.1806, "theoretical_loss": 5.002339946962568, "tokens_seen": 74645504 }, { "epoch": 0.02, "learning_rate": 0.0007897933801404213, "loss": 2.1274, "theoretical_loss": 5.001709753269443, "tokens_seen": 74711040 }, { "epoch": 0.02, "learning_rate": 0.0007897773319959881, "loss": 2.1426, "theoretical_loss": 5.001080266765275, "tokens_seen": 74776576 }, { "epoch": 0.02, "learning_rate": 0.0007897612838515547, "loss": 2.2028, "theoretical_loss": 5.0004514860377185, "tokens_seen": 74842112 }, { "epoch": 0.02, "learning_rate": 0.0007897452357071215, "loss": 2.1624, "theoretical_loss": 4.999823409678481, "tokens_seen": 74907648 }, { "epoch": 0.02, "learning_rate": 0.0007897291875626881, "loss": 2.1102, "theoretical_loss": 4.999196036283308, "tokens_seen": 74973184 }, { "epoch": 0.02, "learning_rate": 0.0007897131394182549, "loss": 2.196, "theoretical_loss": 4.9985693644519715, "tokens_seen": 75038720 }, { "epoch": 0.02, "learning_rate": 0.0007896970912738215, "loss": 2.1068, "theoretical_loss": 4.997943392788251, "tokens_seen": 75104256 }, { "epoch": 0.02, "learning_rate": 0.0007896810431293882, "loss": 2.0444, "theoretical_loss": 4.997318119899922, "tokens_seen": 75169792 }, { "epoch": 0.02, "learning_rate": 0.0007896649949849549, "loss": 1.96, "theoretical_loss": 4.996693544398734, "tokens_seen": 75235328 }, { "epoch": 0.02, "learning_rate": 0.0007896489468405216, "loss": 2.133, "theoretical_loss": 4.996069664900406, "tokens_seen": 75300864 }, { "epoch": 0.02, "objective/train/docs_used": 32032, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.0214736461639404, "objective/train/theoretical_loss": 4.9954464800246035, "objective/train/tokens_used": 95826400, "theoretical_loss": 4.9954464800246035, "tokens_seen": 75366400 }, { "epoch": 0.02, "learning_rate": 0.0007896328986960883, "loss": 2.0347, "theoretical_loss": 4.9954464800246035, "tokens_seen": 75366400 }, { "epoch": 0.02, "learning_rate": 0.000789616850551655, "loss": 2.1474, "theoretical_loss": 4.994823988394928, "tokens_seen": 75431936 }, { "epoch": 0.02, "learning_rate": 0.0007896008024072217, "loss": 2.1111, "theoretical_loss": 4.994202188638901, "tokens_seen": 75497472 }, { "epoch": 0.02, "learning_rate": 0.0007895847542627884, "loss": 2.1961, "theoretical_loss": 4.9935810793879485, "tokens_seen": 75563008 }, { "epoch": 0.02, "learning_rate": 0.0007895687061183551, "loss": 2.1599, "theoretical_loss": 4.992960659277389, "tokens_seen": 75628544 }, { "epoch": 0.02, "learning_rate": 0.0007895526579739218, "loss": 2.1578, "theoretical_loss": 4.992340926946417, "tokens_seen": 75694080 }, { "epoch": 0.02, "learning_rate": 0.0007895366098294884, "loss": 2.041, "theoretical_loss": 4.991721881038091, "tokens_seen": 75759616 }, { "epoch": 0.02, "learning_rate": 0.0007895205616850552, "loss": 2.1556, "theoretical_loss": 4.991103520199314, "tokens_seen": 75825152 }, { "epoch": 0.02, "learning_rate": 0.0007895045135406218, "loss": 2.0935, "theoretical_loss": 4.990485843080824, "tokens_seen": 75890688 }, { "epoch": 0.02, "learning_rate": 0.0007894884653961886, "loss": 2.153, "theoretical_loss": 4.989868848337181, "tokens_seen": 75956224 }, { "epoch": 0.02, "learning_rate": 0.0007894724172517552, "loss": 2.1098, "theoretical_loss": 4.989252534626749, "tokens_seen": 76021760 }, { "epoch": 0.02, "learning_rate": 0.000789456369107322, "loss": 2.2057, "theoretical_loss": 4.988636900611683, "tokens_seen": 76087296 }, { "epoch": 0.02, "learning_rate": 0.0007894403209628886, "loss": 2.1796, "theoretical_loss": 4.988021944957915, "tokens_seen": 76152832 }, { "epoch": 0.02, "learning_rate": 0.0007894242728184554, "loss": 2.1348, "theoretical_loss": 4.987407666335143, "tokens_seen": 76218368 }, { "epoch": 0.02, "learning_rate": 0.000789408224674022, "loss": 2.003, "theoretical_loss": 4.986794063416813, "tokens_seen": 76283904 }, { "epoch": 0.02, "learning_rate": 0.0007893921765295888, "loss": 2.2363, "theoretical_loss": 4.986181134880107, "tokens_seen": 76349440 }, { "epoch": 0.02, "learning_rate": 0.0007893761283851554, "loss": 2.1553, "theoretical_loss": 4.98556887940593, "tokens_seen": 76414976 }, { "epoch": 0.02, "learning_rate": 0.0007893600802407222, "loss": 2.0457, "theoretical_loss": 4.984957295678894, "tokens_seen": 76480512 }, { "epoch": 0.02, "learning_rate": 0.0007893440320962888, "loss": 2.0834, "theoretical_loss": 4.98434638238731, "tokens_seen": 76546048 }, { "epoch": 0.02, "learning_rate": 0.0007893279839518556, "loss": 2.1583, "theoretical_loss": 4.983736138223165, "tokens_seen": 76611584 }, { "epoch": 0.02, "learning_rate": 0.0007893119358074222, "loss": 2.1141, "theoretical_loss": 4.983126561882118, "tokens_seen": 76677120 }, { "epoch": 0.02, "learning_rate": 0.000789295887662989, "loss": 2.1146, "theoretical_loss": 4.982517652063482, "tokens_seen": 76742656 }, { "epoch": 0.02, "learning_rate": 0.0007892798395185556, "loss": 2.0532, "theoretical_loss": 4.981909407470209, "tokens_seen": 76808192 }, { "epoch": 0.02, "learning_rate": 0.0007892637913741224, "loss": 2.1857, "theoretical_loss": 4.981301826808882, "tokens_seen": 76873728 }, { "epoch": 0.02, "learning_rate": 0.000789247743229689, "loss": 2.1357, "theoretical_loss": 4.9806949087896975, "tokens_seen": 76939264 }, { "epoch": 0.02, "learning_rate": 0.0007892316950852558, "loss": 1.9885, "theoretical_loss": 4.980088652126453, "tokens_seen": 77004800 }, { "epoch": 0.02, "learning_rate": 0.0007892156469408224, "loss": 2.1321, "theoretical_loss": 4.9794830555365355, "tokens_seen": 77070336 }, { "epoch": 0.02, "learning_rate": 0.0007891995987963892, "loss": 2.0872, "theoretical_loss": 4.978878117740907, "tokens_seen": 77135872 }, { "epoch": 0.02, "learning_rate": 0.0007891835506519559, "loss": 2.1593, "theoretical_loss": 4.978273837464091, "tokens_seen": 77201408 }, { "epoch": 0.02, "learning_rate": 0.0007891675025075226, "loss": 2.0579, "theoretical_loss": 4.977670213434161, "tokens_seen": 77266944 }, { "epoch": 0.02, "learning_rate": 0.0007891514543630893, "loss": 2.2943, "theoretical_loss": 4.977067244382729, "tokens_seen": 77332480 }, { "epoch": 0.02, "learning_rate": 0.000789135406218656, "loss": 1.9335, "theoretical_loss": 4.976464929044928, "tokens_seen": 77398016 }, { "epoch": 0.02, "learning_rate": 0.0007891193580742227, "loss": 2.0717, "theoretical_loss": 4.975863266159403, "tokens_seen": 77463552 }, { "epoch": 0.02, "learning_rate": 0.0007891033099297894, "loss": 2.0471, "theoretical_loss": 4.975262254468299, "tokens_seen": 77529088 }, { "epoch": 0.02, "learning_rate": 0.0007890872617853561, "loss": 2.07, "theoretical_loss": 4.9746618927172435, "tokens_seen": 77594624 }, { "epoch": 0.02, "learning_rate": 0.0007890712136409228, "loss": 2.1022, "theoretical_loss": 4.9740621796553395, "tokens_seen": 77660160 }, { "epoch": 0.02, "learning_rate": 0.0007890551654964895, "loss": 2.191, "theoretical_loss": 4.973463114035149, "tokens_seen": 77725696 }, { "epoch": 0.02, "learning_rate": 0.0007890391173520562, "loss": 2.0829, "theoretical_loss": 4.972864694612684, "tokens_seen": 77791232 }, { "epoch": 0.02, "learning_rate": 0.000789023069207623, "loss": 2.0328, "theoretical_loss": 4.972266920147391, "tokens_seen": 77856768 }, { "epoch": 0.02, "learning_rate": 0.0007890070210631896, "loss": 2.1021, "theoretical_loss": 4.971669789402139, "tokens_seen": 77922304 }, { "epoch": 0.02, "learning_rate": 0.0007889909729187563, "loss": 2.045, "theoretical_loss": 4.971073301143207, "tokens_seen": 77987840 }, { "epoch": 0.02, "learning_rate": 0.000788974924774323, "loss": 2.1521, "theoretical_loss": 4.970477454140278, "tokens_seen": 78053376 }, { "epoch": 0.02, "learning_rate": 0.0007889588766298897, "loss": 2.1709, "theoretical_loss": 4.9698822471664155, "tokens_seen": 78118912 }, { "epoch": 0.02, "learning_rate": 0.0007889428284854564, "loss": 2.0816, "theoretical_loss": 4.96928767899806, "tokens_seen": 78184448 }, { "epoch": 0.02, "learning_rate": 0.0007889267803410232, "loss": 2.1294, "theoretical_loss": 4.9686937484150135, "tokens_seen": 78249984 }, { "epoch": 0.02, "learning_rate": 0.0007889107321965898, "loss": 2.0653, "theoretical_loss": 4.96810045420043, "tokens_seen": 78315520 }, { "epoch": 0.02, "learning_rate": 0.0007888946840521566, "loss": 2.0877, "theoretical_loss": 4.967507795140797, "tokens_seen": 78381056 }, { "epoch": 0.02, "learning_rate": 0.0007888786359077232, "loss": 2.17, "theoretical_loss": 4.966915770025935, "tokens_seen": 78446592 }, { "epoch": 0.02, "learning_rate": 0.00078886258776329, "loss": 2.1473, "theoretical_loss": 4.966324377648973, "tokens_seen": 78512128 }, { "epoch": 0.02, "learning_rate": 0.0007888465396188566, "loss": 2.1472, "theoretical_loss": 4.965733616806345, "tokens_seen": 78577664 }, { "epoch": 0.02, "objective/train/docs_used": 32979, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.1013200283050537, "objective/train/theoretical_loss": 4.965143486297777, "objective/train/tokens_used": 99103200, "theoretical_loss": 4.965143486297777, "tokens_seen": 78643200 }, { "epoch": 0.02, "learning_rate": 0.0007888304914744234, "loss": 2.0807, "theoretical_loss": 4.965143486297777, "tokens_seen": 78643200 }, { "epoch": 0.02, "learning_rate": 0.00078881444332999, "loss": 2.1694, "theoretical_loss": 4.964553984926271, "tokens_seen": 78708736 }, { "epoch": 0.02, "learning_rate": 0.0007887983951855568, "loss": 2.0942, "theoretical_loss": 4.963965111498098, "tokens_seen": 78774272 }, { "epoch": 0.02, "learning_rate": 0.0007887823470411234, "loss": 2.1027, "theoretical_loss": 4.963376864822784, "tokens_seen": 78839808 }, { "epoch": 0.02, "learning_rate": 0.0007887662988966902, "loss": 2.1018, "theoretical_loss": 4.962789243713102, "tokens_seen": 78905344 }, { "epoch": 0.02, "learning_rate": 0.0007887502507522568, "loss": 2.1184, "theoretical_loss": 4.962202246985054, "tokens_seen": 78970880 }, { "epoch": 0.02, "learning_rate": 0.0007887342026078236, "loss": 2.0848, "theoretical_loss": 4.9616158734578635, "tokens_seen": 79036416 }, { "epoch": 0.02, "learning_rate": 0.0007887181544633902, "loss": 2.0948, "theoretical_loss": 4.961030121953965, "tokens_seen": 79101952 }, { "epoch": 0.02, "learning_rate": 0.000788702106318957, "loss": 2.1181, "theoretical_loss": 4.960444991298992, "tokens_seen": 79167488 }, { "epoch": 0.02, "learning_rate": 0.0007886860581745236, "loss": 2.1234, "theoretical_loss": 4.959860480321762, "tokens_seen": 79233024 }, { "epoch": 0.02, "learning_rate": 0.0007886700100300904, "loss": 2.112, "theoretical_loss": 4.959276587854272, "tokens_seen": 79298560 }, { "epoch": 0.02, "learning_rate": 0.000788653961885657, "loss": 2.1383, "theoretical_loss": 4.958693312731681, "tokens_seen": 79364096 }, { "epoch": 0.02, "learning_rate": 0.0007886379137412238, "loss": 2.1764, "theoretical_loss": 4.9581106537923, "tokens_seen": 79429632 }, { "epoch": 0.02, "learning_rate": 0.0007886218655967904, "loss": 2.0779, "theoretical_loss": 4.957528609877587, "tokens_seen": 79495168 }, { "epoch": 0.02, "learning_rate": 0.0007886058174523572, "loss": 2.0655, "theoretical_loss": 4.9569471798321265, "tokens_seen": 79560704 }, { "epoch": 0.02, "learning_rate": 0.0007885897693079238, "loss": 2.013, "theoretical_loss": 4.9563663625036245, "tokens_seen": 79626240 }, { "epoch": 0.02, "learning_rate": 0.0007885737211634906, "loss": 2.0405, "theoretical_loss": 4.955786156742898, "tokens_seen": 79691776 }, { "epoch": 0.02, "learning_rate": 0.0007885576730190572, "loss": 2.0635, "theoretical_loss": 4.955206561403859, "tokens_seen": 79757312 }, { "epoch": 0.02, "learning_rate": 0.000788541624874624, "loss": 2.1037, "theoretical_loss": 4.954627575343509, "tokens_seen": 79822848 }, { "epoch": 0.02, "learning_rate": 0.0007885255767301906, "loss": 1.9574, "theoretical_loss": 4.954049197421925, "tokens_seen": 79888384 }, { "epoch": 0.02, "learning_rate": 0.0007885095285857574, "loss": 2.1531, "theoretical_loss": 4.953471426502249, "tokens_seen": 79953920 }, { "epoch": 0.02, "learning_rate": 0.000788493480441324, "loss": 2.157, "theoretical_loss": 4.952894261450679, "tokens_seen": 80019456 }, { "epoch": 0.02, "learning_rate": 0.0007884774322968908, "loss": 2.044, "theoretical_loss": 4.952317701136457, "tokens_seen": 80084992 }, { "epoch": 0.02, "learning_rate": 0.0007884613841524574, "loss": 2.1789, "theoretical_loss": 4.9517417444318585, "tokens_seen": 80150528 }, { "epoch": 0.02, "learning_rate": 0.0007884453360080242, "loss": 2.1454, "theoretical_loss": 4.951166390212181, "tokens_seen": 80216064 }, { "epoch": 0.02, "learning_rate": 0.0007884292878635908, "loss": 2.0481, "theoretical_loss": 4.950591637355737, "tokens_seen": 80281600 }, { "epoch": 0.02, "learning_rate": 0.0007884132397191576, "loss": 2.0555, "theoretical_loss": 4.950017484743835, "tokens_seen": 80347136 }, { "epoch": 0.02, "learning_rate": 0.0007883971915747242, "loss": 2.0603, "theoretical_loss": 4.949443931260783, "tokens_seen": 80412672 }, { "epoch": 0.02, "learning_rate": 0.000788381143430291, "loss": 2.0551, "theoretical_loss": 4.948870975793863, "tokens_seen": 80478208 }, { "epoch": 0.02, "learning_rate": 0.0007883650952858576, "loss": 2.1272, "theoretical_loss": 4.9482986172333305, "tokens_seen": 80543744 }, { "epoch": 0.02, "learning_rate": 0.0007883490471414244, "loss": 2.035, "theoretical_loss": 4.9477268544724, "tokens_seen": 80609280 }, { "epoch": 0.02, "learning_rate": 0.000788332998996991, "loss": 2.0542, "theoretical_loss": 4.947155686407239, "tokens_seen": 80674816 }, { "epoch": 0.02, "learning_rate": 0.0007883169508525578, "loss": 2.1393, "theoretical_loss": 4.946585111936951, "tokens_seen": 80740352 }, { "epoch": 0.02, "learning_rate": 0.0007883009027081244, "loss": 2.0737, "theoretical_loss": 4.94601512996357, "tokens_seen": 80805888 }, { "epoch": 0.02, "learning_rate": 0.0007882848545636911, "loss": 1.9408, "theoretical_loss": 4.945445739392049, "tokens_seen": 80871424 }, { "epoch": 0.02, "learning_rate": 0.0007882688064192578, "loss": 1.8858, "theoretical_loss": 4.944876939130252, "tokens_seen": 80936960 }, { "epoch": 0.02, "learning_rate": 0.0007882527582748245, "loss": 2.0729, "theoretical_loss": 4.94430872808894, "tokens_seen": 81002496 }, { "epoch": 0.02, "learning_rate": 0.0007882367101303912, "loss": 2.04, "theoretical_loss": 4.943741105181765, "tokens_seen": 81068032 }, { "epoch": 0.02, "learning_rate": 0.0007882206619859579, "loss": 2.0131, "theoretical_loss": 4.943174069325255, "tokens_seen": 81133568 }, { "epoch": 0.02, "learning_rate": 0.0007882046138415246, "loss": 1.9603, "theoretical_loss": 4.94260761943881, "tokens_seen": 81199104 }, { "epoch": 0.02, "learning_rate": 0.0007881885656970913, "loss": 1.985, "theoretical_loss": 4.942041754444688, "tokens_seen": 81264640 }, { "epoch": 0.02, "learning_rate": 0.000788172517552658, "loss": 2.039, "theoretical_loss": 4.941476473267996, "tokens_seen": 81330176 }, { "epoch": 0.02, "learning_rate": 0.0007881564694082247, "loss": 2.1917, "theoretical_loss": 4.940911774836682, "tokens_seen": 81395712 }, { "epoch": 0.02, "learning_rate": 0.0007881404212637913, "loss": 2.155, "theoretical_loss": 4.940347658081521, "tokens_seen": 81461248 }, { "epoch": 0.02, "learning_rate": 0.0007881243731193581, "loss": 2.0186, "theoretical_loss": 4.939784121936112, "tokens_seen": 81526784 }, { "epoch": 0.02, "learning_rate": 0.0007881083249749247, "loss": 2.0964, "theoretical_loss": 4.9392211653368605, "tokens_seen": 81592320 }, { "epoch": 0.02, "learning_rate": 0.0007880922768304915, "loss": 1.9986, "theoretical_loss": 4.938658787222975, "tokens_seen": 81657856 }, { "epoch": 0.02, "learning_rate": 0.0007880762286860581, "loss": 2.078, "theoretical_loss": 4.938096986536452, "tokens_seen": 81723392 }, { "epoch": 0.02, "learning_rate": 0.0007880601805416249, "loss": 2.0657, "theoretical_loss": 4.937535762222075, "tokens_seen": 81788928 }, { "epoch": 0.02, "learning_rate": 0.0007880441323971915, "loss": 2.111, "theoretical_loss": 4.936975113227394, "tokens_seen": 81854464 }, { "epoch": 0.02, "objective/train/docs_used": 34316, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.1313977241516113, "objective/train/theoretical_loss": 4.936415038502723, "objective/train/tokens_used": 102380000, "theoretical_loss": 4.936415038502723, "tokens_seen": 81920000 }, { "epoch": 0.02, "learning_rate": 0.0007880280842527583, "loss": 2.1647, "theoretical_loss": 4.936415038502723, "tokens_seen": 81920000 }, { "epoch": 0.02, "learning_rate": 0.0007880120361083249, "loss": 2.0975, "theoretical_loss": 4.93585553700113, "tokens_seen": 81985536 }, { "epoch": 0.02, "learning_rate": 0.0007879959879638917, "loss": 2.0226, "theoretical_loss": 4.935296607678426, "tokens_seen": 82051072 }, { "epoch": 0.02, "learning_rate": 0.0007879799398194584, "loss": 2.0912, "theoretical_loss": 4.934738249493156, "tokens_seen": 82116608 }, { "epoch": 0.02, "learning_rate": 0.0007879638916750251, "loss": 2.0794, "theoretical_loss": 4.934180461406591, "tokens_seen": 82182144 }, { "epoch": 0.02, "learning_rate": 0.0007879478435305918, "loss": 2.0681, "theoretical_loss": 4.933623242382714, "tokens_seen": 82247680 }, { "epoch": 0.02, "learning_rate": 0.0007879317953861585, "loss": 2.0694, "theoretical_loss": 4.9330665913882195, "tokens_seen": 82313216 }, { "epoch": 0.02, "learning_rate": 0.0007879157472417252, "loss": 2.0263, "theoretical_loss": 4.932510507392495, "tokens_seen": 82378752 }, { "epoch": 0.02, "learning_rate": 0.0007878996990972919, "loss": 2.0941, "theoretical_loss": 4.931954989367618, "tokens_seen": 82444288 }, { "epoch": 0.03, "learning_rate": 0.0007878836509528586, "loss": 2.1159, "theoretical_loss": 4.931400036288343, "tokens_seen": 82509824 }, { "epoch": 0.03, "learning_rate": 0.0007878676028084253, "loss": 1.965, "theoretical_loss": 4.930845647132097, "tokens_seen": 82575360 }, { "epoch": 0.03, "learning_rate": 0.000787851554663992, "loss": 2.0173, "theoretical_loss": 4.9302918208789634, "tokens_seen": 82640896 }, { "epoch": 0.03, "learning_rate": 0.0007878355065195587, "loss": 2.0566, "theoretical_loss": 4.929738556511681, "tokens_seen": 82706432 }, { "epoch": 0.03, "learning_rate": 0.0007878194583751254, "loss": 2.0425, "theoretical_loss": 4.929185853015629, "tokens_seen": 82771968 }, { "epoch": 0.03, "learning_rate": 0.0007878034102306921, "loss": 2.0891, "theoretical_loss": 4.928633709378822, "tokens_seen": 82837504 }, { "epoch": 0.03, "learning_rate": 0.0007877873620862588, "loss": 2.1192, "theoretical_loss": 4.9280821245918975, "tokens_seen": 82903040 }, { "epoch": 0.03, "learning_rate": 0.0007877713139418255, "loss": 1.8945, "theoretical_loss": 4.92753109764811, "tokens_seen": 82968576 }, { "epoch": 0.03, "learning_rate": 0.0007877552657973922, "loss": 2.1443, "theoretical_loss": 4.92698062754332, "tokens_seen": 83034112 }, { "epoch": 0.03, "learning_rate": 0.0007877392176529589, "loss": 2.263, "theoretical_loss": 4.9264307132759875, "tokens_seen": 83099648 }, { "epoch": 0.03, "learning_rate": 0.0007877231695085256, "loss": 2.0305, "theoretical_loss": 4.92588135384716, "tokens_seen": 83165184 }, { "epoch": 0.03, "learning_rate": 0.0007877071213640923, "loss": 2.0397, "theoretical_loss": 4.925332548260469, "tokens_seen": 83230720 }, { "epoch": 0.03, "learning_rate": 0.000787691073219659, "loss": 1.9951, "theoretical_loss": 4.924784295522113, "tokens_seen": 83296256 }, { "epoch": 0.03, "learning_rate": 0.0007876750250752257, "loss": 2.014, "theoretical_loss": 4.92423659464086, "tokens_seen": 83361792 }, { "epoch": 0.03, "learning_rate": 0.0007876589769307924, "loss": 2.0954, "theoretical_loss": 4.923689444628027, "tokens_seen": 83427328 }, { "epoch": 0.03, "learning_rate": 0.0007876429287863591, "loss": 1.9732, "theoretical_loss": 4.923142844497482, "tokens_seen": 83492864 }, { "epoch": 0.03, "learning_rate": 0.0007876268806419258, "loss": 2.0625, "theoretical_loss": 4.922596793265625, "tokens_seen": 83558400 }, { "epoch": 0.03, "learning_rate": 0.0007876108324974925, "loss": 2.0099, "theoretical_loss": 4.922051289951391, "tokens_seen": 83623936 }, { "epoch": 0.03, "learning_rate": 0.0007875947843530592, "loss": 1.9741, "theoretical_loss": 4.921506333576232, "tokens_seen": 83689472 }, { "epoch": 0.03, "learning_rate": 0.0007875787362086259, "loss": 2.0757, "theoretical_loss": 4.920961923164114, "tokens_seen": 83755008 }, { "epoch": 0.03, "learning_rate": 0.0007875626880641927, "loss": 2.0747, "theoretical_loss": 4.920418057741504, "tokens_seen": 83820544 }, { "epoch": 0.03, "learning_rate": 0.0007875466399197593, "loss": 2.1795, "theoretical_loss": 4.919874736337368, "tokens_seen": 83886080 }, { "epoch": 0.03, "learning_rate": 0.000787530591775326, "loss": 2.0879, "theoretical_loss": 4.9193319579831565, "tokens_seen": 83951616 }, { "epoch": 0.03, "learning_rate": 0.0007875145436308927, "loss": 2.0928, "theoretical_loss": 4.918789721712799, "tokens_seen": 84017152 }, { "epoch": 0.03, "learning_rate": 0.0007874984954864595, "loss": 2.1112, "theoretical_loss": 4.918248026562697, "tokens_seen": 84082688 }, { "epoch": 0.03, "learning_rate": 0.0007874824473420261, "loss": 2.0547, "theoretical_loss": 4.917706871571712, "tokens_seen": 84148224 }, { "epoch": 0.03, "learning_rate": 0.0007874663991975929, "loss": 2.1856, "theoretical_loss": 4.9171662557811615, "tokens_seen": 84213760 }, { "epoch": 0.03, "learning_rate": 0.0007874503510531595, "loss": 2.1379, "theoretical_loss": 4.916626178234807, "tokens_seen": 84279296 }, { "epoch": 0.03, "learning_rate": 0.0007874343029087263, "loss": 2.0925, "theoretical_loss": 4.916086637978851, "tokens_seen": 84344832 }, { "epoch": 0.03, "learning_rate": 0.0007874182547642929, "loss": 2.1199, "theoretical_loss": 4.915547634061921, "tokens_seen": 84410368 }, { "epoch": 0.03, "learning_rate": 0.0007874022066198597, "loss": 2.206, "theoretical_loss": 4.91500916553507, "tokens_seen": 84475904 }, { "epoch": 0.03, "learning_rate": 0.0007873861584754263, "loss": 2.1473, "theoretical_loss": 4.914471231451762, "tokens_seen": 84541440 }, { "epoch": 0.03, "learning_rate": 0.0007873701103309931, "loss": 2.1729, "theoretical_loss": 4.913933830867868, "tokens_seen": 84606976 }, { "epoch": 0.03, "learning_rate": 0.0007873540621865597, "loss": 2.0473, "theoretical_loss": 4.913396962841655, "tokens_seen": 84672512 }, { "epoch": 0.03, "learning_rate": 0.0007873380140421265, "loss": 2.1081, "theoretical_loss": 4.9128606264337815, "tokens_seen": 84738048 }, { "epoch": 0.03, "learning_rate": 0.0007873219658976931, "loss": 2.2149, "theoretical_loss": 4.912324820707285, "tokens_seen": 84803584 }, { "epoch": 0.03, "learning_rate": 0.0007873059177532599, "loss": 2.1657, "theoretical_loss": 4.911789544727581, "tokens_seen": 84869120 }, { "epoch": 0.03, "learning_rate": 0.0007872898696088265, "loss": 2.0486, "theoretical_loss": 4.911254797562448, "tokens_seen": 84934656 }, { "epoch": 0.03, "learning_rate": 0.0007872738214643933, "loss": 2.1763, "theoretical_loss": 4.910720578282021, "tokens_seen": 85000192 }, { "epoch": 0.03, "learning_rate": 0.0007872577733199599, "loss": 2.025, "theoretical_loss": 4.91018688595879, "tokens_seen": 85065728 }, { "epoch": 0.03, "learning_rate": 0.0007872417251755267, "loss": 2.0928, "theoretical_loss": 4.909653719667584, "tokens_seen": 85131264 }, { "epoch": 0.03, "objective/train/docs_used": 35098, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9277470111846924, "objective/train/theoretical_loss": 4.909121078485567, "objective/train/tokens_used": 105656800, "theoretical_loss": 4.909121078485567, "tokens_seen": 85196800 }, { "epoch": 0.03, "learning_rate": 0.0007872256770310933, "loss": 2.1105, "theoretical_loss": 4.909121078485567, "tokens_seen": 85196800 }, { "epoch": 0.03, "learning_rate": 0.0007872096288866601, "loss": 2.0636, "theoretical_loss": 4.908588961492235, "tokens_seen": 85262336 }, { "epoch": 0.03, "learning_rate": 0.0007871935807422267, "loss": 2.1548, "theoretical_loss": 4.908057367769396, "tokens_seen": 85327872 }, { "epoch": 0.03, "learning_rate": 0.0007871775325977935, "loss": 2.146, "theoretical_loss": 4.907526296401175, "tokens_seen": 85393408 }, { "epoch": 0.03, "learning_rate": 0.0007871614844533601, "loss": 2.0305, "theoretical_loss": 4.906995746474001, "tokens_seen": 85458944 }, { "epoch": 0.03, "learning_rate": 0.0007871454363089269, "loss": 2.1158, "theoretical_loss": 4.9064657170766, "tokens_seen": 85524480 }, { "epoch": 0.03, "learning_rate": 0.0007871293881644935, "loss": 2.1002, "theoretical_loss": 4.905936207299984, "tokens_seen": 85590016 }, { "epoch": 0.03, "learning_rate": 0.0007871133400200603, "loss": 2.0993, "theoretical_loss": 4.90540721623745, "tokens_seen": 85655552 }, { "epoch": 0.03, "learning_rate": 0.0007870972918756269, "loss": 1.978, "theoretical_loss": 4.904878742984569, "tokens_seen": 85721088 }, { "epoch": 0.03, "learning_rate": 0.0007870812437311937, "loss": 2.0845, "theoretical_loss": 4.904350786639178, "tokens_seen": 85786624 }, { "epoch": 0.03, "learning_rate": 0.0007870651955867603, "loss": 2.1174, "theoretical_loss": 4.903823346301374, "tokens_seen": 85852160 }, { "epoch": 0.03, "learning_rate": 0.0007870491474423271, "loss": 1.9148, "theoretical_loss": 4.903296421073506, "tokens_seen": 85917696 }, { "epoch": 0.03, "learning_rate": 0.0007870330992978937, "loss": 2.203, "theoretical_loss": 4.902770010060166, "tokens_seen": 85983232 }, { "epoch": 0.03, "learning_rate": 0.0007870170511534605, "loss": 2.1191, "theoretical_loss": 4.902244112368188, "tokens_seen": 86048768 }, { "epoch": 0.03, "learning_rate": 0.0007870010030090271, "loss": 2.1491, "theoretical_loss": 4.901718727106631, "tokens_seen": 86114304 }, { "epoch": 0.03, "learning_rate": 0.0007869849548645939, "loss": 2.1635, "theoretical_loss": 4.90119385338678, "tokens_seen": 86179840 }, { "epoch": 0.03, "learning_rate": 0.0007869689067201605, "loss": 1.9647, "theoretical_loss": 4.900669490322134, "tokens_seen": 86245376 }, { "epoch": 0.03, "learning_rate": 0.0007869528585757273, "loss": 2.0817, "theoretical_loss": 4.900145637028402, "tokens_seen": 86310912 }, { "epoch": 0.03, "learning_rate": 0.0007869368104312939, "loss": 2.1156, "theoretical_loss": 4.899622292623493, "tokens_seen": 86376448 }, { "epoch": 0.03, "learning_rate": 0.0007869207622868607, "loss": 1.9936, "theoretical_loss": 4.8990994562275105, "tokens_seen": 86441984 }, { "epoch": 0.03, "learning_rate": 0.0007869047141424273, "loss": 2.1465, "theoretical_loss": 4.898577126962746, "tokens_seen": 86507520 }, { "epoch": 0.03, "learning_rate": 0.0007868886659979941, "loss": 2.1501, "theoretical_loss": 4.898055303953669, "tokens_seen": 86573056 }, { "epoch": 0.03, "learning_rate": 0.0007868726178535607, "loss": 2.0958, "theoretical_loss": 4.897533986326922, "tokens_seen": 86638592 }, { "epoch": 0.03, "learning_rate": 0.0007868565697091274, "loss": 2.0493, "theoretical_loss": 4.897013173211316, "tokens_seen": 86704128 }, { "epoch": 0.03, "learning_rate": 0.0007868405215646941, "loss": 2.061, "theoretical_loss": 4.896492863737818, "tokens_seen": 86769664 }, { "epoch": 0.03, "learning_rate": 0.0007868244734202608, "loss": 2.1131, "theoretical_loss": 4.895973057039548, "tokens_seen": 86835200 }, { "epoch": 0.03, "learning_rate": 0.0007868084252758275, "loss": 2.0576, "theoretical_loss": 4.89545375225177, "tokens_seen": 86900736 }, { "epoch": 0.03, "learning_rate": 0.0007867923771313942, "loss": 2.0952, "theoretical_loss": 4.894934948511889, "tokens_seen": 86966272 }, { "epoch": 0.03, "learning_rate": 0.0007867763289869609, "loss": 2.0713, "theoretical_loss": 4.894416644959437, "tokens_seen": 87031808 }, { "epoch": 0.03, "learning_rate": 0.0007867602808425276, "loss": 2.1069, "theoretical_loss": 4.893898840736071, "tokens_seen": 87097344 }, { "epoch": 0.03, "learning_rate": 0.0007867442326980943, "loss": 2.0183, "theoretical_loss": 4.8933815349855685, "tokens_seen": 87162880 }, { "epoch": 0.03, "learning_rate": 0.000786728184553661, "loss": 2.1052, "theoretical_loss": 4.892864726853814, "tokens_seen": 87228416 }, { "epoch": 0.03, "learning_rate": 0.0007867121364092277, "loss": 2.0933, "theoretical_loss": 4.892348415488799, "tokens_seen": 87293952 }, { "epoch": 0.03, "learning_rate": 0.0007866960882647944, "loss": 2.0157, "theoretical_loss": 4.891832600040609, "tokens_seen": 87359488 }, { "epoch": 0.03, "learning_rate": 0.0007866800401203611, "loss": 2.1374, "theoretical_loss": 4.891317279661422, "tokens_seen": 87425024 }, { "epoch": 0.03, "learning_rate": 0.0007866639919759278, "loss": 2.0133, "theoretical_loss": 4.890802453505498, "tokens_seen": 87490560 }, { "epoch": 0.03, "learning_rate": 0.0007866479438314945, "loss": 1.901, "theoretical_loss": 4.890288120729174, "tokens_seen": 87556096 }, { "epoch": 0.03, "learning_rate": 0.0007866318956870612, "loss": 2.1402, "theoretical_loss": 4.88977428049086, "tokens_seen": 87621632 }, { "epoch": 0.03, "learning_rate": 0.0007866158475426279, "loss": 1.9311, "theoretical_loss": 4.889260931951026, "tokens_seen": 87687168 }, { "epoch": 0.03, "learning_rate": 0.0007865997993981946, "loss": 2.1391, "theoretical_loss": 4.888748074272201, "tokens_seen": 87752704 }, { "epoch": 0.03, "learning_rate": 0.0007865837512537613, "loss": 2.1003, "theoretical_loss": 4.888235706618964, "tokens_seen": 87818240 }, { "epoch": 0.03, "learning_rate": 0.000786567703109328, "loss": 2.0694, "theoretical_loss": 4.887723828157939, "tokens_seen": 87883776 }, { "epoch": 0.03, "learning_rate": 0.0007865516549648947, "loss": 2.0203, "theoretical_loss": 4.8872124380577855, "tokens_seen": 87949312 }, { "epoch": 0.03, "learning_rate": 0.0007865356068204614, "loss": 2.0587, "theoretical_loss": 4.886701535489195, "tokens_seen": 88014848 }, { "epoch": 0.03, "learning_rate": 0.0007865195586760281, "loss": 2.1354, "theoretical_loss": 4.886191119624883, "tokens_seen": 88080384 }, { "epoch": 0.03, "learning_rate": 0.0007865035105315948, "loss": 2.034, "theoretical_loss": 4.885681189639584, "tokens_seen": 88145920 }, { "epoch": 0.03, "learning_rate": 0.0007864874623871615, "loss": 2.1003, "theoretical_loss": 4.885171744710044, "tokens_seen": 88211456 }, { "epoch": 0.03, "learning_rate": 0.0007864714142427282, "loss": 2.1345, "theoretical_loss": 4.884662784015012, "tokens_seen": 88276992 }, { "epoch": 0.03, "learning_rate": 0.0007864553660982949, "loss": 2.0893, "theoretical_loss": 4.884154306735239, "tokens_seen": 88342528 }, { "epoch": 0.03, "learning_rate": 0.0007864393179538616, "loss": 2.1243, "theoretical_loss": 4.8836463120534646, "tokens_seen": 88408064 }, { "epoch": 0.03, "objective/train/docs_used": 36417, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8406845331192017, "objective/train/theoretical_loss": 4.88313879915442, "objective/train/tokens_used": 108933600, "theoretical_loss": 4.88313879915442, "tokens_seen": 88473600 }, { "epoch": 0.03, "learning_rate": 0.0007864232698094283, "loss": 1.8913, "theoretical_loss": 4.88313879915442, "tokens_seen": 88473600 }, { "epoch": 0.03, "learning_rate": 0.000786407221664995, "loss": 2.1626, "theoretical_loss": 4.882631767224812, "tokens_seen": 88539136 }, { "epoch": 0.03, "learning_rate": 0.0007863911735205617, "loss": 2.0407, "theoretical_loss": 4.882125215453321, "tokens_seen": 88604672 }, { "epoch": 0.03, "learning_rate": 0.0007863751253761284, "loss": 2.0725, "theoretical_loss": 4.881619143030598, "tokens_seen": 88670208 }, { "epoch": 0.03, "learning_rate": 0.0007863590772316951, "loss": 2.007, "theoretical_loss": 4.8811135491492506, "tokens_seen": 88735744 }, { "epoch": 0.03, "learning_rate": 0.0007863430290872618, "loss": 2.0586, "theoretical_loss": 4.880608433003846, "tokens_seen": 88801280 }, { "epoch": 0.03, "learning_rate": 0.0007863269809428285, "loss": 2.0653, "theoretical_loss": 4.880103793790896, "tokens_seen": 88866816 }, { "epoch": 0.03, "learning_rate": 0.0007863109327983952, "loss": 2.0433, "theoretical_loss": 4.87959963070886, "tokens_seen": 88932352 }, { "epoch": 0.03, "learning_rate": 0.000786294884653962, "loss": 2.0326, "theoretical_loss": 4.879095942958127, "tokens_seen": 88997888 }, { "epoch": 0.03, "learning_rate": 0.0007862788365095286, "loss": 2.0379, "theoretical_loss": 4.878592729741024, "tokens_seen": 89063424 }, { "epoch": 0.03, "learning_rate": 0.0007862627883650953, "loss": 2.1523, "theoretical_loss": 4.878089990261797, "tokens_seen": 89128960 }, { "epoch": 0.03, "learning_rate": 0.000786246740220662, "loss": 1.9848, "theoretical_loss": 4.87758772372661, "tokens_seen": 89194496 }, { "epoch": 0.03, "learning_rate": 0.0007862306920762287, "loss": 1.9817, "theoretical_loss": 4.877085929343544, "tokens_seen": 89260032 }, { "epoch": 0.03, "learning_rate": 0.0007862146439317954, "loss": 2.1303, "theoretical_loss": 4.876584606322583, "tokens_seen": 89325568 }, { "epoch": 0.03, "learning_rate": 0.0007861985957873622, "loss": 2.0567, "theoretical_loss": 4.8760837538756086, "tokens_seen": 89391104 }, { "epoch": 0.03, "learning_rate": 0.0007861825476429288, "loss": 1.9851, "theoretical_loss": 4.875583371216404, "tokens_seen": 89456640 }, { "epoch": 0.03, "learning_rate": 0.0007861664994984956, "loss": 2.0458, "theoretical_loss": 4.875083457560633, "tokens_seen": 89522176 }, { "epoch": 0.03, "learning_rate": 0.0007861504513540622, "loss": 1.9532, "theoretical_loss": 4.874584012125847, "tokens_seen": 89587712 }, { "epoch": 0.03, "learning_rate": 0.000786134403209629, "loss": 2.0171, "theoretical_loss": 4.874085034131472, "tokens_seen": 89653248 }, { "epoch": 0.03, "learning_rate": 0.0007861183550651956, "loss": 2.016, "theoretical_loss": 4.873586522798803, "tokens_seen": 89718784 }, { "epoch": 0.03, "learning_rate": 0.0007861023069207624, "loss": 2.0034, "theoretical_loss": 4.873088477351005, "tokens_seen": 89784320 }, { "epoch": 0.03, "learning_rate": 0.000786086258776329, "loss": 2.0347, "theoretical_loss": 4.8725908970130964, "tokens_seen": 89849856 }, { "epoch": 0.03, "learning_rate": 0.0007860702106318958, "loss": 2.0416, "theoretical_loss": 4.872093781011952, "tokens_seen": 89915392 }, { "epoch": 0.03, "learning_rate": 0.0007860541624874624, "loss": 1.9762, "theoretical_loss": 4.871597128576292, "tokens_seen": 89980928 }, { "epoch": 0.03, "learning_rate": 0.0007860381143430292, "loss": 1.9752, "theoretical_loss": 4.8711009389366815, "tokens_seen": 90046464 }, { "epoch": 0.03, "learning_rate": 0.0007860220661985958, "loss": 2.0416, "theoretical_loss": 4.870605211325517, "tokens_seen": 90112000 }, { "epoch": 0.03, "learning_rate": 0.0007860060180541626, "loss": 1.9874, "theoretical_loss": 4.870109944977029, "tokens_seen": 90177536 }, { "epoch": 0.03, "learning_rate": 0.0007859899699097292, "loss": 2.0183, "theoretical_loss": 4.869615139127273, "tokens_seen": 90243072 }, { "epoch": 0.03, "learning_rate": 0.000785973921765296, "loss": 2.079, "theoretical_loss": 4.869120793014117, "tokens_seen": 90308608 }, { "epoch": 0.03, "learning_rate": 0.0007859578736208626, "loss": 2.039, "theoretical_loss": 4.868626905877251, "tokens_seen": 90374144 }, { "epoch": 0.03, "learning_rate": 0.0007859418254764294, "loss": 2.0082, "theoretical_loss": 4.8681334769581674, "tokens_seen": 90439680 }, { "epoch": 0.03, "learning_rate": 0.000785925777331996, "loss": 1.88, "theoretical_loss": 4.867640505500161, "tokens_seen": 90505216 }, { "epoch": 0.03, "learning_rate": 0.0007859097291875628, "loss": 2.0596, "theoretical_loss": 4.8671479907483235, "tokens_seen": 90570752 }, { "epoch": 0.03, "learning_rate": 0.0007858936810431294, "loss": 2.0777, "theoretical_loss": 4.866655931949537, "tokens_seen": 90636288 }, { "epoch": 0.03, "learning_rate": 0.0007858776328986962, "loss": 2.0637, "theoretical_loss": 4.86616432835247, "tokens_seen": 90701824 }, { "epoch": 0.03, "learning_rate": 0.0007858615847542628, "loss": 1.9995, "theoretical_loss": 4.865673179207572, "tokens_seen": 90767360 }, { "epoch": 0.03, "learning_rate": 0.0007858455366098296, "loss": 2.1029, "theoretical_loss": 4.865182483767063, "tokens_seen": 90832896 }, { "epoch": 0.03, "learning_rate": 0.0007858294884653962, "loss": 2.0009, "theoretical_loss": 4.864692241284933, "tokens_seen": 90898432 }, { "epoch": 0.03, "learning_rate": 0.000785813440320963, "loss": 2.0887, "theoretical_loss": 4.864202451016939, "tokens_seen": 90963968 }, { "epoch": 0.03, "learning_rate": 0.0007857973921765296, "loss": 2.0239, "theoretical_loss": 4.863713112220592, "tokens_seen": 91029504 }, { "epoch": 0.03, "learning_rate": 0.0007857813440320964, "loss": 2.0429, "theoretical_loss": 4.8632242241551555, "tokens_seen": 91095040 }, { "epoch": 0.03, "learning_rate": 0.000785765295887663, "loss": 2.0759, "theoretical_loss": 4.862735786081642, "tokens_seen": 91160576 }, { "epoch": 0.03, "learning_rate": 0.0007857492477432298, "loss": 2.014, "theoretical_loss": 4.862247797262806, "tokens_seen": 91226112 }, { "epoch": 0.03, "learning_rate": 0.0007857331995987964, "loss": 2.0947, "theoretical_loss": 4.861760256963136, "tokens_seen": 91291648 }, { "epoch": 0.03, "learning_rate": 0.0007857171514543632, "loss": 2.0521, "theoretical_loss": 4.861273164448854, "tokens_seen": 91357184 }, { "epoch": 0.03, "learning_rate": 0.0007857011033099298, "loss": 1.9563, "theoretical_loss": 4.860786518987906, "tokens_seen": 91422720 }, { "epoch": 0.03, "learning_rate": 0.0007856850551654966, "loss": 2.0038, "theoretical_loss": 4.86030031984996, "tokens_seen": 91488256 }, { "epoch": 0.03, "learning_rate": 0.0007856690070210632, "loss": 1.9585, "theoretical_loss": 4.859814566306397, "tokens_seen": 91553792 }, { "epoch": 0.03, "learning_rate": 0.00078565295887663, "loss": 1.9885, "theoretical_loss": 4.859329257630311, "tokens_seen": 91619328 }, { "epoch": 0.03, "learning_rate": 0.0007856369107321966, "loss": 2.0648, "theoretical_loss": 4.858844393096497, "tokens_seen": 91684864 }, { "epoch": 0.03, "objective/train/docs_used": 37720, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.0330617427825928, "objective/train/theoretical_loss": 4.858359971981454, "objective/train/tokens_used": 112210400, "theoretical_loss": 4.858359971981454, "tokens_seen": 91750400 }, { "epoch": 0.03, "learning_rate": 0.0007856208625877634, "loss": 2.0313, "theoretical_loss": 4.858359971981454, "tokens_seen": 91750400 }, { "epoch": 0.03, "learning_rate": 0.00078560481444333, "loss": 2.1479, "theoretical_loss": 4.85787599356337, "tokens_seen": 91815936 }, { "epoch": 0.03, "learning_rate": 0.0007855887662988968, "loss": 2.0328, "theoretical_loss": 4.857392457122128, "tokens_seen": 91881472 }, { "epoch": 0.03, "learning_rate": 0.0007855727181544634, "loss": 2.1289, "theoretical_loss": 4.856909361939288, "tokens_seen": 91947008 }, { "epoch": 0.03, "learning_rate": 0.0007855566700100302, "loss": 1.9571, "theoretical_loss": 4.856426707298096, "tokens_seen": 92012544 }, { "epoch": 0.03, "learning_rate": 0.0007855406218655968, "loss": 1.9227, "theoretical_loss": 4.8559444924834665, "tokens_seen": 92078080 }, { "epoch": 0.03, "learning_rate": 0.0007855245737211636, "loss": 1.9564, "theoretical_loss": 4.855462716781985, "tokens_seen": 92143616 }, { "epoch": 0.03, "learning_rate": 0.0007855085255767302, "loss": 2.0934, "theoretical_loss": 4.854981379481901, "tokens_seen": 92209152 }, { "epoch": 0.03, "learning_rate": 0.000785492477432297, "loss": 2.0322, "theoretical_loss": 4.854500479873119, "tokens_seen": 92274688 }, { "epoch": 0.03, "learning_rate": 0.0007854764292878636, "loss": 1.9538, "theoretical_loss": 4.854020017247203, "tokens_seen": 92340224 }, { "epoch": 0.03, "learning_rate": 0.0007854603811434303, "loss": 1.9776, "theoretical_loss": 4.853539990897358, "tokens_seen": 92405760 }, { "epoch": 0.03, "learning_rate": 0.000785444332998997, "loss": 2.0646, "theoretical_loss": 4.85306040011844, "tokens_seen": 92471296 }, { "epoch": 0.03, "learning_rate": 0.0007854282848545637, "loss": 1.9781, "theoretical_loss": 4.852581244206938, "tokens_seen": 92536832 }, { "epoch": 0.03, "learning_rate": 0.0007854122367101304, "loss": 2.0651, "theoretical_loss": 4.852102522460975, "tokens_seen": 92602368 }, { "epoch": 0.03, "learning_rate": 0.0007853961885656971, "loss": 1.9221, "theoretical_loss": 4.851624234180306, "tokens_seen": 92667904 }, { "epoch": 0.03, "learning_rate": 0.0007853801404212638, "loss": 2.1273, "theoretical_loss": 4.851146378666305, "tokens_seen": 92733440 }, { "epoch": 0.03, "learning_rate": 0.0007853640922768305, "loss": 1.9866, "theoretical_loss": 4.85066895522197, "tokens_seen": 92798976 }, { "epoch": 0.03, "learning_rate": 0.0007853480441323972, "loss": 2.0728, "theoretical_loss": 4.850191963151908, "tokens_seen": 92864512 }, { "epoch": 0.03, "learning_rate": 0.0007853319959879639, "loss": 2.1431, "theoretical_loss": 4.849715401762337, "tokens_seen": 92930048 }, { "epoch": 0.03, "learning_rate": 0.0007853159478435306, "loss": 2.0067, "theoretical_loss": 4.849239270361078, "tokens_seen": 92995584 }, { "epoch": 0.03, "learning_rate": 0.0007852998996990973, "loss": 1.9941, "theoretical_loss": 4.848763568257554, "tokens_seen": 93061120 }, { "epoch": 0.03, "learning_rate": 0.000785283851554664, "loss": 2.0881, "theoretical_loss": 4.848288294762779, "tokens_seen": 93126656 }, { "epoch": 0.03, "learning_rate": 0.0007852678034102307, "loss": 2.0293, "theoretical_loss": 4.847813449189358, "tokens_seen": 93192192 }, { "epoch": 0.03, "learning_rate": 0.0007852517552657974, "loss": 1.958, "theoretical_loss": 4.847339030851482, "tokens_seen": 93257728 }, { "epoch": 0.03, "learning_rate": 0.0007852357071213641, "loss": 1.9812, "theoretical_loss": 4.84686503906492, "tokens_seen": 93323264 }, { "epoch": 0.03, "learning_rate": 0.0007852196589769308, "loss": 2.1079, "theoretical_loss": 4.846391473147019, "tokens_seen": 93388800 }, { "epoch": 0.03, "learning_rate": 0.0007852036108324975, "loss": 1.9527, "theoretical_loss": 4.8459183324166935, "tokens_seen": 93454336 }, { "epoch": 0.03, "learning_rate": 0.0007851875626880642, "loss": 1.9455, "theoretical_loss": 4.845445616194426, "tokens_seen": 93519872 }, { "epoch": 0.03, "learning_rate": 0.0007851715145436309, "loss": 2.1299, "theoretical_loss": 4.844973323802259, "tokens_seen": 93585408 }, { "epoch": 0.03, "learning_rate": 0.0007851554663991976, "loss": 2.0971, "theoretical_loss": 4.844501454563792, "tokens_seen": 93650944 }, { "epoch": 0.03, "learning_rate": 0.0007851394182547643, "loss": 1.9184, "theoretical_loss": 4.844030007804177, "tokens_seen": 93716480 }, { "epoch": 0.03, "learning_rate": 0.000785123370110331, "loss": 1.9424, "theoretical_loss": 4.843558982850113, "tokens_seen": 93782016 }, { "epoch": 0.03, "learning_rate": 0.0007851073219658977, "loss": 2.0786, "theoretical_loss": 4.84308837902984, "tokens_seen": 93847552 }, { "epoch": 0.03, "learning_rate": 0.0007850912738214644, "loss": 1.9767, "theoretical_loss": 4.842618195673138, "tokens_seen": 93913088 }, { "epoch": 0.03, "learning_rate": 0.0007850752256770311, "loss": 2.0386, "theoretical_loss": 4.84214843211132, "tokens_seen": 93978624 }, { "epoch": 0.03, "learning_rate": 0.0007850591775325978, "loss": 1.9345, "theoretical_loss": 4.841679087677226, "tokens_seen": 94044160 }, { "epoch": 0.03, "learning_rate": 0.0007850431293881645, "loss": 2.1009, "theoretical_loss": 4.841210161705223, "tokens_seen": 94109696 }, { "epoch": 0.03, "learning_rate": 0.0007850270812437312, "loss": 2.0208, "theoretical_loss": 4.840741653531195, "tokens_seen": 94175232 }, { "epoch": 0.03, "learning_rate": 0.0007850110330992979, "loss": 1.8831, "theoretical_loss": 4.840273562492545, "tokens_seen": 94240768 }, { "epoch": 0.03, "learning_rate": 0.0007849949849548646, "loss": 2.0515, "theoretical_loss": 4.839805887928181, "tokens_seen": 94306304 }, { "epoch": 0.03, "learning_rate": 0.0007849789368104313, "loss": 1.9214, "theoretical_loss": 4.839338629178522, "tokens_seen": 94371840 }, { "epoch": 0.03, "learning_rate": 0.000784962888665998, "loss": 1.9477, "theoretical_loss": 4.8388717855854875, "tokens_seen": 94437376 }, { "epoch": 0.03, "learning_rate": 0.0007849468405215647, "loss": 1.927, "theoretical_loss": 4.838405356492494, "tokens_seen": 94502912 }, { "epoch": 0.03, "learning_rate": 0.0007849307923771314, "loss": 1.9606, "theoretical_loss": 4.83793934124445, "tokens_seen": 94568448 }, { "epoch": 0.03, "learning_rate": 0.0007849147442326981, "loss": 1.9475, "theoretical_loss": 4.837473739187754, "tokens_seen": 94633984 }, { "epoch": 0.03, "learning_rate": 0.0007848986960882648, "loss": 2.0574, "theoretical_loss": 4.837008549670285, "tokens_seen": 94699520 }, { "epoch": 0.03, "learning_rate": 0.0007848826479438315, "loss": 2.0748, "theoretical_loss": 4.8365437720414075, "tokens_seen": 94765056 }, { "epoch": 0.03, "learning_rate": 0.0007848665997993982, "loss": 2.0284, "theoretical_loss": 4.836079405651956, "tokens_seen": 94830592 }, { "epoch": 0.03, "learning_rate": 0.0007848505516549649, "loss": 2.039, "theoretical_loss": 4.835615449854238, "tokens_seen": 94896128 }, { "epoch": 0.03, "learning_rate": 0.0007848345035105317, "loss": 1.9871, "theoretical_loss": 4.8351519040020285, "tokens_seen": 94961664 }, { "epoch": 0.03, "objective/train/docs_used": 38548, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.0551602840423584, "objective/train/theoretical_loss": 4.834688767450562, "objective/train/tokens_used": 115487200, "theoretical_loss": 4.834688767450562, "tokens_seen": 95027200 }, { "epoch": 0.03, "learning_rate": 0.0007848184553660983, "loss": 2.0169, "theoretical_loss": 4.834688767450562, "tokens_seen": 95027200 }, { "epoch": 0.03, "learning_rate": 0.000784802407221665, "loss": 1.9129, "theoretical_loss": 4.8342260395565315, "tokens_seen": 95092736 }, { "epoch": 0.03, "learning_rate": 0.0007847863590772317, "loss": 1.9872, "theoretical_loss": 4.833763719678085, "tokens_seen": 95158272 }, { "epoch": 0.03, "learning_rate": 0.0007847703109327985, "loss": 1.954, "theoretical_loss": 4.833301807174822, "tokens_seen": 95223808 }, { "epoch": 0.03, "learning_rate": 0.0007847542627883651, "loss": 2.0831, "theoretical_loss": 4.83284030140778, "tokens_seen": 95289344 }, { "epoch": 0.03, "learning_rate": 0.0007847382146439319, "loss": 1.894, "theoretical_loss": 4.832379201739442, "tokens_seen": 95354880 }, { "epoch": 0.03, "learning_rate": 0.0007847221664994985, "loss": 1.9736, "theoretical_loss": 4.831918507533728, "tokens_seen": 95420416 }, { "epoch": 0.03, "learning_rate": 0.0007847061183550653, "loss": 2.0363, "theoretical_loss": 4.831458218155989, "tokens_seen": 95485952 }, { "epoch": 0.03, "learning_rate": 0.0007846900702106319, "loss": 2.0462, "theoretical_loss": 4.830998332973005, "tokens_seen": 95551488 }, { "epoch": 0.03, "learning_rate": 0.0007846740220661987, "loss": 1.8864, "theoretical_loss": 4.830538851352976, "tokens_seen": 95617024 }, { "epoch": 0.03, "learning_rate": 0.0007846579739217653, "loss": 1.9409, "theoretical_loss": 4.830079772665529, "tokens_seen": 95682560 }, { "epoch": 0.03, "learning_rate": 0.0007846419257773321, "loss": 1.9883, "theoretical_loss": 4.829621096281702, "tokens_seen": 95748096 }, { "epoch": 0.03, "learning_rate": 0.0007846258776328987, "loss": 1.8937, "theoretical_loss": 4.8291628215739415, "tokens_seen": 95813632 }, { "epoch": 0.03, "learning_rate": 0.0007846098294884655, "loss": 1.9911, "theoretical_loss": 4.828704947916108, "tokens_seen": 95879168 }, { "epoch": 0.03, "learning_rate": 0.0007845937813440321, "loss": 1.9991, "theoretical_loss": 4.82824747468346, "tokens_seen": 95944704 }, { "epoch": 0.03, "learning_rate": 0.0007845777331995989, "loss": 1.9115, "theoretical_loss": 4.827790401252658, "tokens_seen": 96010240 }, { "epoch": 0.03, "learning_rate": 0.0007845616850551655, "loss": 1.9385, "theoretical_loss": 4.827333727001756, "tokens_seen": 96075776 }, { "epoch": 0.03, "learning_rate": 0.0007845456369107323, "loss": 1.9522, "theoretical_loss": 4.8268774513101995, "tokens_seen": 96141312 }, { "epoch": 0.03, "learning_rate": 0.0007845295887662989, "loss": 1.8942, "theoretical_loss": 4.82642157355882, "tokens_seen": 96206848 }, { "epoch": 0.03, "learning_rate": 0.0007845135406218657, "loss": 1.9496, "theoretical_loss": 4.825966093129834, "tokens_seen": 96272384 }, { "epoch": 0.03, "learning_rate": 0.0007844974924774323, "loss": 1.9805, "theoretical_loss": 4.825511009406833, "tokens_seen": 96337920 }, { "epoch": 0.03, "learning_rate": 0.0007844814443329991, "loss": 1.9984, "theoretical_loss": 4.825056321774788, "tokens_seen": 96403456 }, { "epoch": 0.03, "learning_rate": 0.0007844653961885657, "loss": 1.9623, "theoretical_loss": 4.8246020296200385, "tokens_seen": 96468992 }, { "epoch": 0.03, "learning_rate": 0.0007844493480441325, "loss": 1.9433, "theoretical_loss": 4.82414813233029, "tokens_seen": 96534528 }, { "epoch": 0.03, "learning_rate": 0.0007844332998996991, "loss": 2.0465, "theoretical_loss": 4.823694629294609, "tokens_seen": 96600064 }, { "epoch": 0.03, "learning_rate": 0.0007844172517552659, "loss": 2.0518, "theoretical_loss": 4.823241519903428, "tokens_seen": 96665600 }, { "epoch": 0.03, "learning_rate": 0.0007844012036108325, "loss": 1.9394, "theoretical_loss": 4.8227888035485265, "tokens_seen": 96731136 }, { "epoch": 0.03, "learning_rate": 0.0007843851554663993, "loss": 1.8898, "theoretical_loss": 4.82233647962304, "tokens_seen": 96796672 }, { "epoch": 0.03, "learning_rate": 0.000784369107321966, "loss": 1.9072, "theoretical_loss": 4.821884547521449, "tokens_seen": 96862208 }, { "epoch": 0.03, "learning_rate": 0.0007843530591775327, "loss": 2.0767, "theoretical_loss": 4.821433006639578, "tokens_seen": 96927744 }, { "epoch": 0.03, "learning_rate": 0.0007843370110330994, "loss": 1.9428, "theoretical_loss": 4.820981856374589, "tokens_seen": 96993280 }, { "epoch": 0.03, "learning_rate": 0.0007843209628886661, "loss": 1.9907, "theoretical_loss": 4.8205310961249825, "tokens_seen": 97058816 }, { "epoch": 0.03, "learning_rate": 0.0007843049147442328, "loss": 1.9943, "theoretical_loss": 4.820080725290589, "tokens_seen": 97124352 }, { "epoch": 0.03, "learning_rate": 0.0007842888665997995, "loss": 1.9588, "theoretical_loss": 4.819630743272566, "tokens_seen": 97189888 }, { "epoch": 0.03, "learning_rate": 0.0007842728184553662, "loss": 2.0534, "theoretical_loss": 4.819181149473396, "tokens_seen": 97255424 }, { "epoch": 0.03, "learning_rate": 0.0007842567703109329, "loss": 2.0432, "theoretical_loss": 4.81873194329688, "tokens_seen": 97320960 }, { "epoch": 0.03, "learning_rate": 0.0007842407221664996, "loss": 2.0024, "theoretical_loss": 4.81828312414814, "tokens_seen": 97386496 }, { "epoch": 0.03, "learning_rate": 0.0007842246740220663, "loss": 2.0566, "theoretical_loss": 4.817834691433602, "tokens_seen": 97452032 }, { "epoch": 0.03, "learning_rate": 0.0007842086258776329, "loss": 1.9204, "theoretical_loss": 4.81738664456101, "tokens_seen": 97517568 }, { "epoch": 0.03, "learning_rate": 0.0007841925777331997, "loss": 1.8443, "theoretical_loss": 4.816938982939407, "tokens_seen": 97583104 }, { "epoch": 0.03, "learning_rate": 0.0007841765295887663, "loss": 2.0024, "theoretical_loss": 4.816491705979138, "tokens_seen": 97648640 }, { "epoch": 0.03, "learning_rate": 0.0007841604814443331, "loss": 2.0781, "theoretical_loss": 4.816044813091848, "tokens_seen": 97714176 }, { "epoch": 0.03, "learning_rate": 0.0007841444332998997, "loss": 2.0516, "theoretical_loss": 4.815598303690473, "tokens_seen": 97779712 }, { "epoch": 0.03, "learning_rate": 0.0007841283851554665, "loss": 1.8457, "theoretical_loss": 4.8151521771892405, "tokens_seen": 97845248 }, { "epoch": 0.03, "learning_rate": 0.0007841123370110331, "loss": 1.9794, "theoretical_loss": 4.814706433003665, "tokens_seen": 97910784 }, { "epoch": 0.03, "learning_rate": 0.0007840962888665999, "loss": 1.988, "theoretical_loss": 4.8142610705505415, "tokens_seen": 97976320 }, { "epoch": 0.03, "learning_rate": 0.0007840802407221665, "loss": 1.9506, "theoretical_loss": 4.813816089247945, "tokens_seen": 98041856 }, { "epoch": 0.03, "learning_rate": 0.0007840641925777332, "loss": 2.0677, "theoretical_loss": 4.813371488515227, "tokens_seen": 98107392 }, { "epoch": 0.03, "learning_rate": 0.0007840481444332999, "loss": 1.8749, "theoretical_loss": 4.812927267773008, "tokens_seen": 98172928 }, { "epoch": 0.03, "learning_rate": 0.0007840320962888666, "loss": 2.0423, "theoretical_loss": 4.812483426443181, "tokens_seen": 98238464 }, { "epoch": 0.03, "objective/train/docs_used": 39774, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.065314292907715, "objective/train/theoretical_loss": 4.812039963948898, "objective/train/tokens_used": 118764000, "theoretical_loss": 4.812039963948898, "tokens_seen": 98304000 }, { "epoch": 0.03, "learning_rate": 0.0007840160481444333, "loss": 1.9389, "theoretical_loss": 4.812039963948898, "tokens_seen": 98304000 }, { "epoch": 0.03, "learning_rate": 0.000784, "loss": 2.0512, "theoretical_loss": 4.811596879714575, "tokens_seen": 98369536 }, { "epoch": 0.03, "learning_rate": 0.0007839839518555667, "loss": 2.0692, "theoretical_loss": 4.811154173165886, "tokens_seen": 98435072 }, { "epoch": 0.03, "learning_rate": 0.0007839679037111334, "loss": 2.0087, "theoretical_loss": 4.810711843729758, "tokens_seen": 98500608 }, { "epoch": 0.03, "learning_rate": 0.0007839518555667001, "loss": 1.9157, "theoretical_loss": 4.810269890834364, "tokens_seen": 98566144 }, { "epoch": 0.03, "learning_rate": 0.0007839358074222668, "loss": 1.8269, "theoretical_loss": 4.809828313909129, "tokens_seen": 98631680 }, { "epoch": 0.03, "learning_rate": 0.0007839197592778335, "loss": 1.9956, "theoretical_loss": 4.809387112384721, "tokens_seen": 98697216 }, { "epoch": 0.03, "learning_rate": 0.0007839037111334002, "loss": 1.9715, "theoretical_loss": 4.808946285693043, "tokens_seen": 98762752 }, { "epoch": 0.03, "learning_rate": 0.0007838876629889669, "loss": 1.8838, "theoretical_loss": 4.808505833267237, "tokens_seen": 98828288 }, { "epoch": 0.03, "learning_rate": 0.0007838716148445336, "loss": 1.8904, "theoretical_loss": 4.808065754541676, "tokens_seen": 98893824 }, { "epoch": 0.03, "learning_rate": 0.0007838555667001003, "loss": 1.89, "theoretical_loss": 4.807626048951965, "tokens_seen": 98959360 }, { "epoch": 0.03, "learning_rate": 0.000783839518555667, "loss": 1.9809, "theoretical_loss": 4.807186715934931, "tokens_seen": 99024896 }, { "epoch": 0.03, "learning_rate": 0.0007838234704112337, "loss": 1.8864, "theoretical_loss": 4.806747754928622, "tokens_seen": 99090432 }, { "epoch": 0.03, "learning_rate": 0.0007838074222668004, "loss": 1.8412, "theoretical_loss": 4.806309165372311, "tokens_seen": 99155968 }, { "epoch": 0.03, "learning_rate": 0.0007837913741223671, "loss": 1.8538, "theoretical_loss": 4.805870946706479, "tokens_seen": 99221504 }, { "epoch": 0.03, "learning_rate": 0.0007837753259779338, "loss": 2.0107, "theoretical_loss": 4.805433098372822, "tokens_seen": 99287040 }, { "epoch": 0.03, "learning_rate": 0.0007837592778335005, "loss": 2.0084, "theoretical_loss": 4.8049956198142425, "tokens_seen": 99352576 }, { "epoch": 0.03, "learning_rate": 0.0007837432296890672, "loss": 1.9323, "theoretical_loss": 4.804558510474852, "tokens_seen": 99418112 }, { "epoch": 0.03, "learning_rate": 0.0007837271815446339, "loss": 1.9652, "theoretical_loss": 4.804121769799959, "tokens_seen": 99483648 }, { "epoch": 0.03, "learning_rate": 0.0007837111334002006, "loss": 1.8399, "theoretical_loss": 4.8036853972360705, "tokens_seen": 99549184 }, { "epoch": 0.03, "learning_rate": 0.0007836950852557673, "loss": 1.9356, "theoretical_loss": 4.80324939223089, "tokens_seen": 99614720 }, { "epoch": 0.03, "learning_rate": 0.000783679037111334, "loss": 2.0503, "theoretical_loss": 4.802813754233311, "tokens_seen": 99680256 }, { "epoch": 0.03, "learning_rate": 0.0007836629889669007, "loss": 1.9646, "theoretical_loss": 4.802378482693417, "tokens_seen": 99745792 }, { "epoch": 0.03, "learning_rate": 0.0007836469408224674, "loss": 1.9134, "theoretical_loss": 4.801943577062472, "tokens_seen": 99811328 }, { "epoch": 0.03, "learning_rate": 0.0007836308926780341, "loss": 1.9831, "theoretical_loss": 4.801509036792925, "tokens_seen": 99876864 }, { "epoch": 0.03, "learning_rate": 0.0007836148445336008, "loss": 2.1101, "theoretical_loss": 4.801074861338401, "tokens_seen": 99942400 }, { "epoch": 0.03, "learning_rate": 0.0007835987963891675, "loss": 1.9731, "theoretical_loss": 4.800641050153703, "tokens_seen": 100007936 }, { "epoch": 0.03, "learning_rate": 0.0007835827482447342, "loss": 2.0708, "theoretical_loss": 4.800207602694801, "tokens_seen": 100073472 }, { "epoch": 0.03, "learning_rate": 0.000783566700100301, "loss": 1.9291, "theoretical_loss": 4.7997745184188325, "tokens_seen": 100139008 }, { "epoch": 0.03, "learning_rate": 0.0007835506519558676, "loss": 2.0162, "theoretical_loss": 4.799341796784106, "tokens_seen": 100204544 }, { "epoch": 0.03, "learning_rate": 0.0007835346038114343, "loss": 2.0161, "theoretical_loss": 4.798909437250086, "tokens_seen": 100270080 }, { "epoch": 0.03, "learning_rate": 0.000783518555667001, "loss": 1.872, "theoretical_loss": 4.798477439277397, "tokens_seen": 100335616 }, { "epoch": 0.03, "learning_rate": 0.0007835025075225677, "loss": 1.9359, "theoretical_loss": 4.7980458023278185, "tokens_seen": 100401152 }, { "epoch": 0.03, "learning_rate": 0.0007834864593781344, "loss": 2.046, "theoretical_loss": 4.797614525864282, "tokens_seen": 100466688 }, { "epoch": 0.03, "learning_rate": 0.0007834704112337011, "loss": 2.0471, "theoretical_loss": 4.797183609350869, "tokens_seen": 100532224 }, { "epoch": 0.03, "learning_rate": 0.0007834543630892678, "loss": 1.9752, "theoretical_loss": 4.796753052252802, "tokens_seen": 100597760 }, { "epoch": 0.03, "learning_rate": 0.0007834383149448346, "loss": 1.8555, "theoretical_loss": 4.79632285403645, "tokens_seen": 100663296 }, { "epoch": 0.03, "learning_rate": 0.0007834222668004012, "loss": 1.9888, "theoretical_loss": 4.795893014169323, "tokens_seen": 100728832 }, { "epoch": 0.03, "learning_rate": 0.000783406218655968, "loss": 1.9027, "theoretical_loss": 4.795463532120058, "tokens_seen": 100794368 }, { "epoch": 0.03, "learning_rate": 0.0007833901705115346, "loss": 1.9115, "theoretical_loss": 4.795034407358435, "tokens_seen": 100859904 }, { "epoch": 0.03, "learning_rate": 0.0007833741223671014, "loss": 1.9345, "theoretical_loss": 4.794605639355357, "tokens_seen": 100925440 }, { "epoch": 0.03, "learning_rate": 0.000783358074222668, "loss": 1.9462, "theoretical_loss": 4.794177227582855, "tokens_seen": 100990976 }, { "epoch": 0.03, "learning_rate": 0.0007833420260782348, "loss": 1.9011, "theoretical_loss": 4.793749171514085, "tokens_seen": 101056512 }, { "epoch": 0.03, "learning_rate": 0.0007833259779338014, "loss": 1.8762, "theoretical_loss": 4.793321470623321, "tokens_seen": 101122048 }, { "epoch": 0.03, "learning_rate": 0.0007833099297893682, "loss": 1.9244, "theoretical_loss": 4.792894124385955, "tokens_seen": 101187584 }, { "epoch": 0.03, "learning_rate": 0.0007832938816449348, "loss": 1.9741, "theoretical_loss": 4.792467132278494, "tokens_seen": 101253120 }, { "epoch": 0.03, "learning_rate": 0.0007832778335005016, "loss": 1.9348, "theoretical_loss": 4.792040493778553, "tokens_seen": 101318656 }, { "epoch": 0.03, "learning_rate": 0.0007832617853560682, "loss": 2.0157, "theoretical_loss": 4.791614208364859, "tokens_seen": 101384192 }, { "epoch": 0.03, "learning_rate": 0.000783245737211635, "loss": 1.9452, "theoretical_loss": 4.791188275517239, "tokens_seen": 101449728 }, { "epoch": 0.03, "learning_rate": 0.0007832296890672016, "loss": 1.9476, "theoretical_loss": 4.7907626947166255, "tokens_seen": 101515264 }, { "epoch": 0.03, "objective/train/docs_used": 40787, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9380860328674316, "objective/train/theoretical_loss": 4.790337465445049, "objective/train/tokens_used": 122040800, "theoretical_loss": 4.790337465445049, "tokens_seen": 101580800 }, { "epoch": 0.03, "learning_rate": 0.0007832136409227684, "loss": 1.9874, "theoretical_loss": 4.790337465445049, "tokens_seen": 101580800 }, { "epoch": 0.03, "learning_rate": 0.000783197592778335, "loss": 2.06, "theoretical_loss": 4.7899125871856345, "tokens_seen": 101646336 }, { "epoch": 0.03, "learning_rate": 0.0007831815446339018, "loss": 1.8105, "theoretical_loss": 4.789488059422599, "tokens_seen": 101711872 }, { "epoch": 0.03, "learning_rate": 0.0007831654964894685, "loss": 2.0042, "theoretical_loss": 4.789063881641251, "tokens_seen": 101777408 }, { "epoch": 0.03, "learning_rate": 0.0007831494483450352, "loss": 2.0481, "theoretical_loss": 4.7886400533279865, "tokens_seen": 101842944 }, { "epoch": 0.03, "learning_rate": 0.0007831334002006019, "loss": 1.9965, "theoretical_loss": 4.788216573970281, "tokens_seen": 101908480 }, { "epoch": 0.03, "learning_rate": 0.0007831173520561686, "loss": 1.9324, "theoretical_loss": 4.787793443056694, "tokens_seen": 101974016 }, { "epoch": 0.03, "learning_rate": 0.0007831013039117353, "loss": 1.8745, "theoretical_loss": 4.787370660076862, "tokens_seen": 102039552 }, { "epoch": 0.03, "learning_rate": 0.000783085255767302, "loss": 2.0481, "theoretical_loss": 4.786948224521495, "tokens_seen": 102105088 }, { "epoch": 0.03, "learning_rate": 0.0007830692076228687, "loss": 1.9234, "theoretical_loss": 4.786526135882378, "tokens_seen": 102170624 }, { "epoch": 0.03, "learning_rate": 0.0007830531594784354, "loss": 2.0611, "theoretical_loss": 4.786104393652359, "tokens_seen": 102236160 }, { "epoch": 0.03, "learning_rate": 0.0007830371113340021, "loss": 1.9584, "theoretical_loss": 4.785682997325358, "tokens_seen": 102301696 }, { "epoch": 0.03, "learning_rate": 0.0007830210631895688, "loss": 1.9192, "theoretical_loss": 4.7852619463963535, "tokens_seen": 102367232 }, { "epoch": 0.03, "learning_rate": 0.0007830050150451355, "loss": 1.8914, "theoretical_loss": 4.784841240361389, "tokens_seen": 102432768 }, { "epoch": 0.03, "learning_rate": 0.0007829889669007022, "loss": 1.9203, "theoretical_loss": 4.784420878717558, "tokens_seen": 102498304 }, { "epoch": 0.03, "learning_rate": 0.0007829729187562689, "loss": 1.9652, "theoretical_loss": 4.7840008609630145, "tokens_seen": 102563840 }, { "epoch": 0.03, "learning_rate": 0.0007829568706118356, "loss": 1.9889, "theoretical_loss": 4.783581186596962, "tokens_seen": 102629376 }, { "epoch": 0.03, "learning_rate": 0.0007829408224674023, "loss": 1.8846, "theoretical_loss": 4.78316185511965, "tokens_seen": 102694912 }, { "epoch": 0.03, "learning_rate": 0.000782924774322969, "loss": 2.0007, "theoretical_loss": 4.782742866032379, "tokens_seen": 102760448 }, { "epoch": 0.03, "learning_rate": 0.0007829087261785357, "loss": 1.8957, "theoretical_loss": 4.782324218837486, "tokens_seen": 102825984 }, { "epoch": 0.03, "learning_rate": 0.0007828926780341024, "loss": 1.9462, "theoretical_loss": 4.781905913038351, "tokens_seen": 102891520 }, { "epoch": 0.03, "learning_rate": 0.0007828766298896691, "loss": 1.9478, "theoretical_loss": 4.781487948139393, "tokens_seen": 102957056 }, { "epoch": 0.03, "learning_rate": 0.0007828605817452358, "loss": 1.9648, "theoretical_loss": 4.7810703236460625, "tokens_seen": 103022592 }, { "epoch": 0.03, "learning_rate": 0.0007828445336008025, "loss": 1.9738, "theoretical_loss": 4.780653039064843, "tokens_seen": 103088128 }, { "epoch": 0.03, "learning_rate": 0.0007828284854563692, "loss": 1.9539, "theoretical_loss": 4.780236093903245, "tokens_seen": 103153664 }, { "epoch": 0.03, "learning_rate": 0.0007828124373119358, "loss": 1.8898, "theoretical_loss": 4.779819487669805, "tokens_seen": 103219200 }, { "epoch": 0.03, "learning_rate": 0.0007827963891675026, "loss": 1.8978, "theoretical_loss": 4.779403219874085, "tokens_seen": 103284736 }, { "epoch": 0.03, "learning_rate": 0.0007827803410230692, "loss": 1.97, "theoretical_loss": 4.778987290026665, "tokens_seen": 103350272 }, { "epoch": 0.03, "learning_rate": 0.000782764292878636, "loss": 1.9551, "theoretical_loss": 4.778571697639142, "tokens_seen": 103415808 }, { "epoch": 0.03, "learning_rate": 0.0007827482447342026, "loss": 1.8563, "theoretical_loss": 4.778156442224132, "tokens_seen": 103481344 }, { "epoch": 0.03, "learning_rate": 0.0007827321965897694, "loss": 2.0819, "theoretical_loss": 4.7777415232952585, "tokens_seen": 103546880 }, { "epoch": 0.03, "learning_rate": 0.000782716148445336, "loss": 1.9497, "theoretical_loss": 4.777326940367155, "tokens_seen": 103612416 }, { "epoch": 0.03, "learning_rate": 0.0007827001003009028, "loss": 2.0545, "theoretical_loss": 4.776912692955463, "tokens_seen": 103677952 }, { "epoch": 0.03, "learning_rate": 0.0007826840521564694, "loss": 1.9213, "theoretical_loss": 4.776498780576826, "tokens_seen": 103743488 }, { "epoch": 0.03, "learning_rate": 0.0007826680040120361, "loss": 1.9726, "theoretical_loss": 4.776085202748893, "tokens_seen": 103809024 }, { "epoch": 0.03, "learning_rate": 0.0007826519558676028, "loss": 1.8898, "theoretical_loss": 4.775671958990305, "tokens_seen": 103874560 }, { "epoch": 0.03, "learning_rate": 0.0007826359077231695, "loss": 2.004, "theoretical_loss": 4.775259048820702, "tokens_seen": 103940096 }, { "epoch": 0.03, "learning_rate": 0.0007826198595787362, "loss": 1.9216, "theoretical_loss": 4.7748464717607195, "tokens_seen": 104005632 }, { "epoch": 0.03, "learning_rate": 0.0007826038114343029, "loss": 1.8736, "theoretical_loss": 4.774434227331979, "tokens_seen": 104071168 }, { "epoch": 0.03, "learning_rate": 0.0007825877632898696, "loss": 1.9748, "theoretical_loss": 4.774022315057092, "tokens_seen": 104136704 }, { "epoch": 0.03, "learning_rate": 0.0007825717151454363, "loss": 1.9511, "theoretical_loss": 4.773610734459654, "tokens_seen": 104202240 }, { "epoch": 0.03, "learning_rate": 0.000782555667001003, "loss": 1.8595, "theoretical_loss": 4.773199485064243, "tokens_seen": 104267776 }, { "epoch": 0.03, "learning_rate": 0.0007825396188565697, "loss": 1.8144, "theoretical_loss": 4.772788566396418, "tokens_seen": 104333312 }, { "epoch": 0.03, "learning_rate": 0.0007825235707121364, "loss": 1.9711, "theoretical_loss": 4.772377977982714, "tokens_seen": 104398848 }, { "epoch": 0.03, "learning_rate": 0.0007825075225677031, "loss": 1.8884, "theoretical_loss": 4.771967719350641, "tokens_seen": 104464384 }, { "epoch": 0.03, "learning_rate": 0.0007824914744232698, "loss": 1.9437, "theoretical_loss": 4.77155779002868, "tokens_seen": 104529920 }, { "epoch": 0.03, "learning_rate": 0.0007824754262788365, "loss": 1.8921, "theoretical_loss": 4.771148189546282, "tokens_seen": 104595456 }, { "epoch": 0.03, "learning_rate": 0.0007824593781344032, "loss": 2.0112, "theoretical_loss": 4.770738917433864, "tokens_seen": 104660992 }, { "epoch": 0.03, "learning_rate": 0.0007824433299899699, "loss": 1.9376, "theoretical_loss": 4.770329973222809, "tokens_seen": 104726528 }, { "epoch": 0.03, "learning_rate": 0.0007824272818455366, "loss": 1.966, "theoretical_loss": 4.769921356445458, "tokens_seen": 104792064 }, { "epoch": 0.03, "objective/train/docs_used": 42078, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.965983510017395, "objective/train/theoretical_loss": 4.769513066635114, "objective/train/tokens_used": 125317600, "theoretical_loss": 4.769513066635114, "tokens_seen": 104857600 }, { "epoch": 0.03, "learning_rate": 0.0007824112337011033, "loss": 1.9, "theoretical_loss": 4.769513066635114, "tokens_seen": 104857600 }, { "epoch": 0.03, "learning_rate": 0.00078239518555667, "loss": 1.9222, "theoretical_loss": 4.769105103326036, "tokens_seen": 104923136 }, { "epoch": 0.03, "learning_rate": 0.0007823791374122367, "loss": 2.0369, "theoretical_loss": 4.768697466053435, "tokens_seen": 104988672 }, { "epoch": 0.03, "learning_rate": 0.0007823630892678034, "loss": 1.9214, "theoretical_loss": 4.768290154353474, "tokens_seen": 105054208 }, { "epoch": 0.03, "learning_rate": 0.0007823470411233701, "loss": 1.9252, "theoretical_loss": 4.767883167763267, "tokens_seen": 105119744 }, { "epoch": 0.03, "learning_rate": 0.0007823309929789368, "loss": 1.9184, "theoretical_loss": 4.76747650582087, "tokens_seen": 105185280 }, { "epoch": 0.03, "learning_rate": 0.0007823149448345035, "loss": 1.8375, "theoretical_loss": 4.767070168065285, "tokens_seen": 105250816 }, { "epoch": 0.03, "learning_rate": 0.0007822988966900702, "loss": 1.8323, "theoretical_loss": 4.766664154036456, "tokens_seen": 105316352 }, { "epoch": 0.03, "learning_rate": 0.0007822828485456369, "loss": 1.7604, "theoretical_loss": 4.766258463275265, "tokens_seen": 105381888 }, { "epoch": 0.03, "learning_rate": 0.0007822668004012036, "loss": 1.9705, "theoretical_loss": 4.7658530953235285, "tokens_seen": 105447424 }, { "epoch": 0.03, "learning_rate": 0.0007822507522567703, "loss": 1.9077, "theoretical_loss": 4.765448049723998, "tokens_seen": 105512960 }, { "epoch": 0.03, "learning_rate": 0.000782234704112337, "loss": 1.7903, "theoretical_loss": 4.765043326020358, "tokens_seen": 105578496 }, { "epoch": 0.03, "learning_rate": 0.0007822186559679037, "loss": 1.9913, "theoretical_loss": 4.764638923757217, "tokens_seen": 105644032 }, { "epoch": 0.03, "learning_rate": 0.0007822026078234704, "loss": 1.8003, "theoretical_loss": 4.764234842480114, "tokens_seen": 105709568 }, { "epoch": 0.03, "learning_rate": 0.0007821865596790371, "loss": 1.8746, "theoretical_loss": 4.7638310817355105, "tokens_seen": 105775104 }, { "epoch": 0.03, "learning_rate": 0.0007821705115346038, "loss": 1.9363, "theoretical_loss": 4.763427641070789, "tokens_seen": 105840640 }, { "epoch": 0.03, "learning_rate": 0.0007821544633901705, "loss": 1.8989, "theoretical_loss": 4.76302452003425, "tokens_seen": 105906176 }, { "epoch": 0.03, "learning_rate": 0.0007821384152457372, "loss": 1.9954, "theoretical_loss": 4.7626217181751125, "tokens_seen": 105971712 }, { "epoch": 0.03, "learning_rate": 0.0007821223671013039, "loss": 1.9583, "theoretical_loss": 4.762219235043508, "tokens_seen": 106037248 }, { "epoch": 0.03, "learning_rate": 0.0007821063189568706, "loss": 1.9582, "theoretical_loss": 4.761817070190481, "tokens_seen": 106102784 }, { "epoch": 0.03, "learning_rate": 0.0007820902708124373, "loss": 2.018, "theoretical_loss": 4.761415223167982, "tokens_seen": 106168320 }, { "epoch": 0.03, "learning_rate": 0.000782074222668004, "loss": 2.0643, "theoretical_loss": 4.761013693528874, "tokens_seen": 106233856 }, { "epoch": 0.03, "learning_rate": 0.0007820581745235707, "loss": 1.9534, "theoretical_loss": 4.760612480826917, "tokens_seen": 106299392 }, { "epoch": 0.03, "learning_rate": 0.0007820421263791375, "loss": 1.9656, "theoretical_loss": 4.76021158461678, "tokens_seen": 106364928 }, { "epoch": 0.03, "learning_rate": 0.0007820260782347041, "loss": 1.9398, "theoretical_loss": 4.759811004454027, "tokens_seen": 106430464 }, { "epoch": 0.03, "learning_rate": 0.0007820100300902709, "loss": 1.9891, "theoretical_loss": 4.759410739895122, "tokens_seen": 106496000 }, { "epoch": 0.03, "learning_rate": 0.0007819939819458375, "loss": 1.9332, "theoretical_loss": 4.759010790497422, "tokens_seen": 106561536 }, { "epoch": 0.03, "learning_rate": 0.0007819779338014043, "loss": 1.9756, "theoretical_loss": 4.758611155819178, "tokens_seen": 106627072 }, { "epoch": 0.03, "learning_rate": 0.000781961885656971, "loss": 1.9532, "theoretical_loss": 4.758211835419531, "tokens_seen": 106692608 }, { "epoch": 0.03, "learning_rate": 0.0007819458375125377, "loss": 1.8169, "theoretical_loss": 4.757812828858508, "tokens_seen": 106758144 }, { "epoch": 0.03, "learning_rate": 0.0007819297893681044, "loss": 1.8261, "theoretical_loss": 4.757414135697024, "tokens_seen": 106823680 }, { "epoch": 0.03, "learning_rate": 0.0007819137412236711, "loss": 1.9218, "theoretical_loss": 4.757015755496877, "tokens_seen": 106889216 }, { "epoch": 0.03, "learning_rate": 0.0007818976930792378, "loss": 1.7847, "theoretical_loss": 4.756617687820743, "tokens_seen": 106954752 }, { "epoch": 0.03, "learning_rate": 0.0007818816449348045, "loss": 1.9165, "theoretical_loss": 4.756219932232181, "tokens_seen": 107020288 }, { "epoch": 0.03, "learning_rate": 0.0007818655967903712, "loss": 1.9031, "theoretical_loss": 4.755822488295621, "tokens_seen": 107085824 }, { "epoch": 0.03, "learning_rate": 0.0007818495486459379, "loss": 2.0325, "theoretical_loss": 4.755425355576373, "tokens_seen": 107151360 }, { "epoch": 0.03, "learning_rate": 0.0007818335005015046, "loss": 1.9744, "theoretical_loss": 4.755028533640614, "tokens_seen": 107216896 }, { "epoch": 0.03, "learning_rate": 0.0007818174523570713, "loss": 1.9273, "theoretical_loss": 4.754632022055392, "tokens_seen": 107282432 }, { "epoch": 0.03, "learning_rate": 0.000781801404212638, "loss": 1.9457, "theoretical_loss": 4.754235820388624, "tokens_seen": 107347968 }, { "epoch": 0.03, "learning_rate": 0.0007817853560682047, "loss": 1.7709, "theoretical_loss": 4.7538399282090875, "tokens_seen": 107413504 }, { "epoch": 0.03, "learning_rate": 0.0007817693079237714, "loss": 1.9709, "theoretical_loss": 4.753444345086428, "tokens_seen": 107479040 }, { "epoch": 0.03, "learning_rate": 0.0007817532597793381, "loss": 1.8569, "theoretical_loss": 4.753049070591146, "tokens_seen": 107544576 }, { "epoch": 0.03, "learning_rate": 0.0007817372116349048, "loss": 1.8751, "theoretical_loss": 4.752654104294605, "tokens_seen": 107610112 }, { "epoch": 0.03, "learning_rate": 0.0007817211634904715, "loss": 1.9358, "theoretical_loss": 4.752259445769022, "tokens_seen": 107675648 }, { "epoch": 0.03, "learning_rate": 0.0007817051153460382, "loss": 1.8664, "theoretical_loss": 4.751865094587465, "tokens_seen": 107741184 }, { "epoch": 0.03, "learning_rate": 0.0007816890672016049, "loss": 1.8228, "theoretical_loss": 4.75147105032386, "tokens_seen": 107806720 }, { "epoch": 0.03, "learning_rate": 0.0007816730190571716, "loss": 1.8293, "theoretical_loss": 4.751077312552976, "tokens_seen": 107872256 }, { "epoch": 0.03, "learning_rate": 0.0007816569709127383, "loss": 1.8754, "theoretical_loss": 4.750683880850433, "tokens_seen": 107937792 }, { "epoch": 0.03, "learning_rate": 0.000781640922768305, "loss": 1.9043, "theoretical_loss": 4.750290754792694, "tokens_seen": 108003328 }, { "epoch": 0.03, "learning_rate": 0.0007816248746238717, "loss": 1.8329, "theoretical_loss": 4.749897933957065, "tokens_seen": 108068864 }, { "epoch": 0.03, "objective/train/docs_used": 42949, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9812370538711548, "objective/train/theoretical_loss": 4.749505417921692, "objective/train/tokens_used": 128594400, "theoretical_loss": 4.749505417921692, "tokens_seen": 108134400 }, { "epoch": 0.03, "learning_rate": 0.0007816088264794384, "loss": 1.989, "theoretical_loss": 4.749505417921692, "tokens_seen": 108134400 }, { "epoch": 0.03, "learning_rate": 0.0007815927783350051, "loss": 1.9188, "theoretical_loss": 4.7491132062655605, "tokens_seen": 108199936 }, { "epoch": 0.03, "learning_rate": 0.0007815767301905718, "loss": 1.8283, "theoretical_loss": 4.748721298568491, "tokens_seen": 108265472 }, { "epoch": 0.03, "learning_rate": 0.0007815606820461385, "loss": 2.0035, "theoretical_loss": 4.748329694411137, "tokens_seen": 108331008 }, { "epoch": 0.03, "learning_rate": 0.0007815446339017052, "loss": 1.9027, "theoretical_loss": 4.747938393374987, "tokens_seen": 108396544 }, { "epoch": 0.03, "learning_rate": 0.0007815285857572719, "loss": 1.8085, "theoretical_loss": 4.747547395042355, "tokens_seen": 108462080 }, { "epoch": 0.03, "learning_rate": 0.0007815125376128386, "loss": 1.8566, "theoretical_loss": 4.747156698996388, "tokens_seen": 108527616 }, { "epoch": 0.03, "learning_rate": 0.0007814964894684053, "loss": 1.8354, "theoretical_loss": 4.746766304821053, "tokens_seen": 108593152 }, { "epoch": 0.03, "learning_rate": 0.000781480441323972, "loss": 1.98, "theoretical_loss": 4.746376212101142, "tokens_seen": 108658688 }, { "epoch": 0.03, "learning_rate": 0.0007814643931795387, "loss": 1.98, "theoretical_loss": 4.745986420422267, "tokens_seen": 108724224 }, { "epoch": 0.03, "learning_rate": 0.0007814483450351054, "loss": 1.8918, "theoretical_loss": 4.745596929370863, "tokens_seen": 108789760 }, { "epoch": 0.03, "learning_rate": 0.0007814322968906721, "loss": 1.9303, "theoretical_loss": 4.745207738534177, "tokens_seen": 108855296 }, { "epoch": 0.03, "learning_rate": 0.0007814162487462388, "loss": 1.8991, "theoretical_loss": 4.744818847500274, "tokens_seen": 108920832 }, { "epoch": 0.03, "learning_rate": 0.0007814002006018055, "loss": 1.9216, "theoretical_loss": 4.744430255858029, "tokens_seen": 108986368 }, { "epoch": 0.03, "learning_rate": 0.0007813841524573721, "loss": 1.8368, "theoretical_loss": 4.744041963197132, "tokens_seen": 109051904 }, { "epoch": 0.03, "learning_rate": 0.0007813681043129389, "loss": 1.9503, "theoretical_loss": 4.743653969108076, "tokens_seen": 109117440 }, { "epoch": 0.03, "learning_rate": 0.0007813520561685055, "loss": 1.8052, "theoretical_loss": 4.743266273182163, "tokens_seen": 109182976 }, { "epoch": 0.03, "learning_rate": 0.0007813360080240723, "loss": 1.8667, "theoretical_loss": 4.742878875011501, "tokens_seen": 109248512 }, { "epoch": 0.03, "learning_rate": 0.0007813199598796389, "loss": 1.8948, "theoretical_loss": 4.742491774188997, "tokens_seen": 109314048 }, { "epoch": 0.03, "learning_rate": 0.0007813039117352057, "loss": 2.0147, "theoretical_loss": 4.742104970308359, "tokens_seen": 109379584 }, { "epoch": 0.03, "learning_rate": 0.0007812878635907723, "loss": 1.8349, "theoretical_loss": 4.741718462964095, "tokens_seen": 109445120 }, { "epoch": 0.03, "learning_rate": 0.000781271815446339, "loss": 2.0293, "theoretical_loss": 4.741332251751509, "tokens_seen": 109510656 }, { "epoch": 0.03, "learning_rate": 0.0007812557673019057, "loss": 1.9101, "theoretical_loss": 4.740946336266697, "tokens_seen": 109576192 }, { "epoch": 0.03, "learning_rate": 0.0007812397191574724, "loss": 1.9097, "theoretical_loss": 4.740560716106549, "tokens_seen": 109641728 }, { "epoch": 0.03, "learning_rate": 0.0007812236710130391, "loss": 1.9329, "theoretical_loss": 4.740175390868742, "tokens_seen": 109707264 }, { "epoch": 0.03, "learning_rate": 0.0007812076228686058, "loss": 1.8663, "theoretical_loss": 4.739790360151744, "tokens_seen": 109772800 }, { "epoch": 0.03, "learning_rate": 0.0007811915747241725, "loss": 1.9846, "theoretical_loss": 4.73940562355481, "tokens_seen": 109838336 }, { "epoch": 0.03, "learning_rate": 0.0007811755265797392, "loss": 1.8642, "theoretical_loss": 4.739021180677976, "tokens_seen": 109903872 }, { "epoch": 0.03, "learning_rate": 0.0007811594784353059, "loss": 1.8875, "theoretical_loss": 4.73863703112206, "tokens_seen": 109969408 }, { "epoch": 0.03, "learning_rate": 0.0007811434302908726, "loss": 1.8295, "theoretical_loss": 4.738253174488662, "tokens_seen": 110034944 }, { "epoch": 0.03, "learning_rate": 0.0007811273821464393, "loss": 1.8563, "theoretical_loss": 4.73786961038016, "tokens_seen": 110100480 }, { "epoch": 0.03, "learning_rate": 0.000781111334002006, "loss": 1.9474, "theoretical_loss": 4.737486338399704, "tokens_seen": 110166016 }, { "epoch": 0.03, "learning_rate": 0.0007810952858575727, "loss": 2.0176, "theoretical_loss": 4.737103358151225, "tokens_seen": 110231552 }, { "epoch": 0.03, "learning_rate": 0.0007810792377131394, "loss": 1.8906, "theoretical_loss": 4.7367206692394195, "tokens_seen": 110297088 }, { "epoch": 0.03, "learning_rate": 0.0007810631895687061, "loss": 1.8809, "theoretical_loss": 4.736338271269759, "tokens_seen": 110362624 }, { "epoch": 0.03, "learning_rate": 0.0007810471414242728, "loss": 1.931, "theoretical_loss": 4.735956163848478, "tokens_seen": 110428160 }, { "epoch": 0.03, "learning_rate": 0.0007810310932798395, "loss": 1.968, "theoretical_loss": 4.735574346582582, "tokens_seen": 110493696 }, { "epoch": 0.03, "learning_rate": 0.0007810150451354062, "loss": 1.9077, "theoretical_loss": 4.735192819079838, "tokens_seen": 110559232 }, { "epoch": 0.03, "learning_rate": 0.0007809989969909729, "loss": 1.8817, "theoretical_loss": 4.734811580948779, "tokens_seen": 110624768 }, { "epoch": 0.03, "learning_rate": 0.0007809829488465396, "loss": 1.8448, "theoretical_loss": 4.734430631798692, "tokens_seen": 110690304 }, { "epoch": 0.03, "learning_rate": 0.0007809669007021063, "loss": 1.9445, "theoretical_loss": 4.734049971239628, "tokens_seen": 110755840 }, { "epoch": 0.03, "learning_rate": 0.000780950852557673, "loss": 1.9162, "theoretical_loss": 4.7336695988823925, "tokens_seen": 110821376 }, { "epoch": 0.03, "learning_rate": 0.0007809348044132397, "loss": 1.9709, "theoretical_loss": 4.733289514338546, "tokens_seen": 110886912 }, { "epoch": 0.03, "learning_rate": 0.0007809187562688064, "loss": 1.8657, "theoretical_loss": 4.7329097172204, "tokens_seen": 110952448 }, { "epoch": 0.03, "learning_rate": 0.0007809027081243731, "loss": 1.7885, "theoretical_loss": 4.73253020714102, "tokens_seen": 111017984 }, { "epoch": 0.03, "learning_rate": 0.0007808866599799398, "loss": 1.9659, "theoretical_loss": 4.732150983714217, "tokens_seen": 111083520 }, { "epoch": 0.03, "learning_rate": 0.0007808706118355065, "loss": 1.8298, "theoretical_loss": 4.731772046554552, "tokens_seen": 111149056 }, { "epoch": 0.03, "learning_rate": 0.0007808545636910732, "loss": 1.9417, "theoretical_loss": 4.731393395277331, "tokens_seen": 111214592 }, { "epoch": 0.03, "learning_rate": 0.0007808385155466399, "loss": 1.8618, "theoretical_loss": 4.731015029498598, "tokens_seen": 111280128 }, { "epoch": 0.03, "learning_rate": 0.0007808224674022066, "loss": 1.9002, "theoretical_loss": 4.730636948835148, "tokens_seen": 111345664 }, { "epoch": 0.03, "objective/train/docs_used": 44130, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9857747554779053, "objective/train/theoretical_loss": 4.730259152904507, "objective/train/tokens_used": 131871200, "theoretical_loss": 4.730259152904507, "tokens_seen": 111411200 }, { "epoch": 0.03, "learning_rate": 0.0007808064192577733, "loss": 1.9456, "theoretical_loss": 4.730259152904507, "tokens_seen": 111411200 }, { "epoch": 0.03, "learning_rate": 0.00078079037111334, "loss": 1.7301, "theoretical_loss": 4.7298816413249405, "tokens_seen": 111476736 }, { "epoch": 0.03, "learning_rate": 0.0007807743229689067, "loss": 1.7904, "theoretical_loss": 4.729504413715454, "tokens_seen": 111542272 }, { "epoch": 0.03, "learning_rate": 0.0007807582748244735, "loss": 1.8232, "theoretical_loss": 4.729127469695783, "tokens_seen": 111607808 }, { "epoch": 0.03, "learning_rate": 0.0007807422266800401, "loss": 1.8824, "theoretical_loss": 4.728750808886394, "tokens_seen": 111673344 }, { "epoch": 0.03, "learning_rate": 0.0007807261785356069, "loss": 1.9541, "theoretical_loss": 4.72837443090849, "tokens_seen": 111738880 }, { "epoch": 0.03, "learning_rate": 0.0007807101303911736, "loss": 1.9042, "theoretical_loss": 4.727998335383996, "tokens_seen": 111804416 }, { "epoch": 0.03, "learning_rate": 0.0007806940822467403, "loss": 1.8814, "theoretical_loss": 4.727622521935563, "tokens_seen": 111869952 }, { "epoch": 0.03, "learning_rate": 0.000780678034102307, "loss": 1.8453, "theoretical_loss": 4.7272469901865755, "tokens_seen": 111935488 }, { "epoch": 0.03, "learning_rate": 0.0007806619859578737, "loss": 1.8953, "theoretical_loss": 4.726871739761132, "tokens_seen": 112001024 }, { "epoch": 0.03, "learning_rate": 0.0007806459378134404, "loss": 1.8651, "theoretical_loss": 4.726496770284056, "tokens_seen": 112066560 }, { "epoch": 0.03, "learning_rate": 0.0007806298896690071, "loss": 1.9383, "theoretical_loss": 4.726122081380891, "tokens_seen": 112132096 }, { "epoch": 0.03, "learning_rate": 0.0007806138415245738, "loss": 1.8434, "theoretical_loss": 4.725747672677894, "tokens_seen": 112197632 }, { "epoch": 0.03, "learning_rate": 0.0007805977933801405, "loss": 1.9197, "theoretical_loss": 4.725373543802043, "tokens_seen": 112263168 }, { "epoch": 0.03, "learning_rate": 0.0007805817452357072, "loss": 1.9461, "theoretical_loss": 4.724999694381027, "tokens_seen": 112328704 }, { "epoch": 0.03, "learning_rate": 0.0007805656970912739, "loss": 1.858, "theoretical_loss": 4.724626124043247, "tokens_seen": 112394240 }, { "epoch": 0.03, "learning_rate": 0.0007805496489468406, "loss": 1.9208, "theoretical_loss": 4.724252832417817, "tokens_seen": 112459776 }, { "epoch": 0.03, "learning_rate": 0.0007805336008024073, "loss": 1.7769, "theoretical_loss": 4.723879819134558, "tokens_seen": 112525312 }, { "epoch": 0.03, "learning_rate": 0.000780517552657974, "loss": 1.807, "theoretical_loss": 4.7235070838239945, "tokens_seen": 112590848 }, { "epoch": 0.03, "learning_rate": 0.0007805015045135407, "loss": 1.9201, "theoretical_loss": 4.723134626117363, "tokens_seen": 112656384 }, { "epoch": 0.03, "learning_rate": 0.0007804854563691074, "loss": 1.8909, "theoretical_loss": 4.722762445646598, "tokens_seen": 112721920 }, { "epoch": 0.03, "learning_rate": 0.0007804694082246741, "loss": 1.9641, "theoretical_loss": 4.722390542044339, "tokens_seen": 112787456 }, { "epoch": 0.03, "learning_rate": 0.0007804533600802408, "loss": 1.8542, "theoretical_loss": 4.722018914943924, "tokens_seen": 112852992 }, { "epoch": 0.03, "learning_rate": 0.0007804373119358075, "loss": 1.7947, "theoretical_loss": 4.721647563979385, "tokens_seen": 112918528 }, { "epoch": 0.03, "learning_rate": 0.0007804212637913742, "loss": 1.7628, "theoretical_loss": 4.72127648878546, "tokens_seen": 112984064 }, { "epoch": 0.03, "learning_rate": 0.0007804052156469409, "loss": 2.0437, "theoretical_loss": 4.720905688997572, "tokens_seen": 113049600 }, { "epoch": 0.03, "learning_rate": 0.0007803891675025076, "loss": 1.8696, "theoretical_loss": 4.720535164251842, "tokens_seen": 113115136 }, { "epoch": 0.03, "learning_rate": 0.0007803731193580743, "loss": 2.0169, "theoretical_loss": 4.720164914185084, "tokens_seen": 113180672 }, { "epoch": 0.03, "learning_rate": 0.000780357071213641, "loss": 1.9562, "theoretical_loss": 4.719794938434794, "tokens_seen": 113246208 }, { "epoch": 0.03, "learning_rate": 0.0007803410230692077, "loss": 1.8761, "theoretical_loss": 4.7194252366391645, "tokens_seen": 113311744 }, { "epoch": 0.03, "learning_rate": 0.0007803249749247744, "loss": 1.8482, "theoretical_loss": 4.719055808437068, "tokens_seen": 113377280 }, { "epoch": 0.03, "learning_rate": 0.0007803089267803411, "loss": 1.8214, "theoretical_loss": 4.718686653468065, "tokens_seen": 113442816 }, { "epoch": 0.03, "learning_rate": 0.0007802928786359078, "loss": 1.89, "theoretical_loss": 4.718317771372398, "tokens_seen": 113508352 }, { "epoch": 0.03, "learning_rate": 0.0007802768304914745, "loss": 2.0138, "theoretical_loss": 4.7179491617909886, "tokens_seen": 113573888 }, { "epoch": 0.03, "learning_rate": 0.0007802607823470412, "loss": 1.8962, "theoretical_loss": 4.717580824365439, "tokens_seen": 113639424 }, { "epoch": 0.03, "learning_rate": 0.0007802447342026079, "loss": 1.9767, "theoretical_loss": 4.71721275873803, "tokens_seen": 113704960 }, { "epoch": 0.03, "learning_rate": 0.0007802286860581746, "loss": 1.8079, "theoretical_loss": 4.716844964551717, "tokens_seen": 113770496 }, { "epoch": 0.03, "learning_rate": 0.0007802126379137413, "loss": 1.9601, "theoretical_loss": 4.716477441450131, "tokens_seen": 113836032 }, { "epoch": 0.03, "learning_rate": 0.000780196589769308, "loss": 1.8804, "theoretical_loss": 4.716110189077575, "tokens_seen": 113901568 }, { "epoch": 0.03, "learning_rate": 0.0007801805416248747, "loss": 1.9204, "theoretical_loss": 4.715743207079022, "tokens_seen": 113967104 }, { "epoch": 0.03, "learning_rate": 0.0007801644934804414, "loss": 1.8374, "theoretical_loss": 4.715376495100118, "tokens_seen": 114032640 }, { "epoch": 0.03, "learning_rate": 0.0007801484453360081, "loss": 1.9237, "theoretical_loss": 4.7150100527871714, "tokens_seen": 114098176 }, { "epoch": 0.03, "learning_rate": 0.0007801323971915748, "loss": 1.8361, "theoretical_loss": 4.714643879787161, "tokens_seen": 114163712 }, { "epoch": 0.03, "learning_rate": 0.0007801163490471415, "loss": 1.9508, "theoretical_loss": 4.714277975747729, "tokens_seen": 114229248 }, { "epoch": 0.03, "learning_rate": 0.0007801003009027082, "loss": 1.8377, "theoretical_loss": 4.7139123403171785, "tokens_seen": 114294784 }, { "epoch": 0.03, "learning_rate": 0.0007800842527582749, "loss": 1.771, "theoretical_loss": 4.713546973144476, "tokens_seen": 114360320 }, { "epoch": 0.03, "learning_rate": 0.0007800682046138416, "loss": 1.7296, "theoretical_loss": 4.713181873879248, "tokens_seen": 114425856 }, { "epoch": 0.03, "learning_rate": 0.0007800521564694083, "loss": 1.9301, "theoretical_loss": 4.712817042171776, "tokens_seen": 114491392 }, { "epoch": 0.03, "learning_rate": 0.000780036108324975, "loss": 1.804, "theoretical_loss": 4.712452477673001, "tokens_seen": 114556928 }, { "epoch": 0.03, "learning_rate": 0.0007800200601805417, "loss": 1.7899, "theoretical_loss": 4.712088180034517, "tokens_seen": 114622464 }, { "epoch": 0.03, "objective/train/docs_used": 45100, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7656515836715698, "objective/train/theoretical_loss": 4.711724148908571, "objective/train/tokens_used": 135148000, "theoretical_loss": 4.711724148908571, "tokens_seen": 114688000 }, { "epoch": 0.03, "learning_rate": 0.0007800040120361084, "loss": 1.7773, "theoretical_loss": 4.711724148908571, "tokens_seen": 114688000 }, { "epoch": 0.03, "learning_rate": 0.000779987963891675, "loss": 1.8698, "theoretical_loss": 4.711360383948064, "tokens_seen": 114753536 }, { "epoch": 0.03, "learning_rate": 0.0007799719157472418, "loss": 1.81, "theoretical_loss": 4.710996884806542, "tokens_seen": 114819072 }, { "epoch": 0.03, "learning_rate": 0.0007799558676028084, "loss": 1.8963, "theoretical_loss": 4.710633651138206, "tokens_seen": 114884608 }, { "epoch": 0.03, "learning_rate": 0.0007799398194583752, "loss": 2.0256, "theoretical_loss": 4.710270682597898, "tokens_seen": 114950144 }, { "epoch": 0.03, "learning_rate": 0.0007799237713139418, "loss": 1.9627, "theoretical_loss": 4.709907978841109, "tokens_seen": 115015680 }, { "epoch": 0.03, "learning_rate": 0.0007799077231695086, "loss": 1.9976, "theoretical_loss": 4.709545539523971, "tokens_seen": 115081216 }, { "epoch": 0.03, "learning_rate": 0.0007798916750250752, "loss": 1.7934, "theoretical_loss": 4.709183364303258, "tokens_seen": 115146752 }, { "epoch": 0.03, "learning_rate": 0.000779875626880642, "loss": 1.7876, "theoretical_loss": 4.708821452836388, "tokens_seen": 115212288 }, { "epoch": 0.03, "learning_rate": 0.0007798595787362086, "loss": 1.8766, "theoretical_loss": 4.708459804781414, "tokens_seen": 115277824 }, { "epoch": 0.03, "learning_rate": 0.0007798435305917753, "loss": 1.8795, "theoretical_loss": 4.708098419797031, "tokens_seen": 115343360 }, { "epoch": 0.03, "learning_rate": 0.000779827482447342, "loss": 1.8992, "theoretical_loss": 4.707737297542563, "tokens_seen": 115408896 }, { "epoch": 0.03, "learning_rate": 0.0007798114343029087, "loss": 1.7714, "theoretical_loss": 4.707376437677974, "tokens_seen": 115474432 }, { "epoch": 0.04, "learning_rate": 0.0007797953861584754, "loss": 1.8703, "theoretical_loss": 4.707015839863859, "tokens_seen": 115539968 }, { "epoch": 0.04, "learning_rate": 0.0007797793380140421, "loss": 1.8995, "theoretical_loss": 4.706655503761443, "tokens_seen": 115605504 }, { "epoch": 0.04, "learning_rate": 0.0007797632898696088, "loss": 1.8673, "theoretical_loss": 4.706295429032584, "tokens_seen": 115671040 }, { "epoch": 0.04, "learning_rate": 0.0007797472417251755, "loss": 1.8975, "theoretical_loss": 4.705935615339763, "tokens_seen": 115736576 }, { "epoch": 0.04, "learning_rate": 0.0007797311935807422, "loss": 1.964, "theoretical_loss": 4.705576062346094, "tokens_seen": 115802112 }, { "epoch": 0.04, "learning_rate": 0.0007797151454363089, "loss": 1.7302, "theoretical_loss": 4.70521676971531, "tokens_seen": 115867648 }, { "epoch": 0.04, "learning_rate": 0.0007796990972918756, "loss": 1.8957, "theoretical_loss": 4.704857737111773, "tokens_seen": 115933184 }, { "epoch": 0.04, "learning_rate": 0.0007796830491474423, "loss": 1.8279, "theoretical_loss": 4.704498964200463, "tokens_seen": 115998720 }, { "epoch": 0.04, "learning_rate": 0.000779667001003009, "loss": 1.9176, "theoretical_loss": 4.70414045064698, "tokens_seen": 116064256 }, { "epoch": 0.04, "learning_rate": 0.0007796509528585757, "loss": 1.9368, "theoretical_loss": 4.703782196117549, "tokens_seen": 116129792 }, { "epoch": 0.04, "learning_rate": 0.0007796349047141424, "loss": 1.9024, "theoretical_loss": 4.703424200279007, "tokens_seen": 116195328 }, { "epoch": 0.04, "learning_rate": 0.0007796188565697091, "loss": 1.7334, "theoretical_loss": 4.703066462798809, "tokens_seen": 116260864 }, { "epoch": 0.04, "learning_rate": 0.0007796028084252758, "loss": 2.0384, "theoretical_loss": 4.7027089833450235, "tokens_seen": 116326400 }, { "epoch": 0.04, "learning_rate": 0.0007795867602808425, "loss": 1.8372, "theoretical_loss": 4.7023517615863355, "tokens_seen": 116391936 }, { "epoch": 0.04, "learning_rate": 0.0007795707121364092, "loss": 1.7742, "theoretical_loss": 4.701994797192038, "tokens_seen": 116457472 }, { "epoch": 0.04, "learning_rate": 0.000779554663991976, "loss": 1.977, "theoretical_loss": 4.701638089832034, "tokens_seen": 116523008 }, { "epoch": 0.04, "learning_rate": 0.0007795386158475426, "loss": 1.9297, "theoretical_loss": 4.701281639176839, "tokens_seen": 116588544 }, { "epoch": 0.04, "learning_rate": 0.0007795225677031094, "loss": 1.8724, "theoretical_loss": 4.700925444897575, "tokens_seen": 116654080 }, { "epoch": 0.04, "learning_rate": 0.000779506519558676, "loss": 1.7618, "theoretical_loss": 4.700569506665965, "tokens_seen": 116719616 }, { "epoch": 0.04, "learning_rate": 0.0007794904714142428, "loss": 1.8056, "theoretical_loss": 4.700213824154341, "tokens_seen": 116785152 }, { "epoch": 0.04, "learning_rate": 0.0007794744232698094, "loss": 1.8923, "theoretical_loss": 4.699858397035637, "tokens_seen": 116850688 }, { "epoch": 0.04, "learning_rate": 0.0007794583751253762, "loss": 1.9056, "theoretical_loss": 4.699503224983388, "tokens_seen": 116916224 }, { "epoch": 0.04, "learning_rate": 0.0007794423269809428, "loss": 1.9701, "theoretical_loss": 4.699148307671729, "tokens_seen": 116981760 }, { "epoch": 0.04, "learning_rate": 0.0007794262788365096, "loss": 1.8863, "theoretical_loss": 4.698793644775394, "tokens_seen": 117047296 }, { "epoch": 0.04, "learning_rate": 0.0007794102306920762, "loss": 1.9355, "theoretical_loss": 4.698439235969713, "tokens_seen": 117112832 }, { "epoch": 0.04, "learning_rate": 0.000779394182547643, "loss": 1.9582, "theoretical_loss": 4.698085080930612, "tokens_seen": 117178368 }, { "epoch": 0.04, "learning_rate": 0.0007793781344032096, "loss": 1.8616, "theoretical_loss": 4.697731179334614, "tokens_seen": 117243904 }, { "epoch": 0.04, "learning_rate": 0.0007793620862587764, "loss": 1.7266, "theoretical_loss": 4.69737753085883, "tokens_seen": 117309440 }, { "epoch": 0.04, "learning_rate": 0.000779346038114343, "loss": 1.9112, "theoretical_loss": 4.697024135180968, "tokens_seen": 117374976 }, { "epoch": 0.04, "learning_rate": 0.0007793299899699098, "loss": 1.8236, "theoretical_loss": 4.696670991979321, "tokens_seen": 117440512 }, { "epoch": 0.04, "learning_rate": 0.0007793139418254765, "loss": 1.8411, "theoretical_loss": 4.696318100932773, "tokens_seen": 117506048 }, { "epoch": 0.04, "learning_rate": 0.0007792978936810432, "loss": 1.7059, "theoretical_loss": 4.695965461720796, "tokens_seen": 117571584 }, { "epoch": 0.04, "learning_rate": 0.0007792818455366099, "loss": 1.822, "theoretical_loss": 4.695613074023446, "tokens_seen": 117637120 }, { "epoch": 0.04, "learning_rate": 0.0007792657973921766, "loss": 1.894, "theoretical_loss": 4.695260937521364, "tokens_seen": 117702656 }, { "epoch": 0.04, "learning_rate": 0.0007792497492477433, "loss": 1.7513, "theoretical_loss": 4.694909051895775, "tokens_seen": 117768192 }, { "epoch": 0.04, "learning_rate": 0.00077923370110331, "loss": 1.8627, "theoretical_loss": 4.694557416828484, "tokens_seen": 117833728 }, { "epoch": 0.04, "learning_rate": 0.0007792176529588767, "loss": 1.7953, "theoretical_loss": 4.694206032001879, "tokens_seen": 117899264 }, { "epoch": 0.04, "objective/train/docs_used": 46257, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9019111394882202, "objective/train/theoretical_loss": 4.693854897098923, "objective/train/tokens_used": 138424800, "theoretical_loss": 4.693854897098923, "tokens_seen": 117964800 }, { "epoch": 0.04, "learning_rate": 0.0007792016048144434, "loss": 1.9417, "theoretical_loss": 4.693854897098923, "tokens_seen": 117964800 }, { "epoch": 0.04, "learning_rate": 0.0007791855566700101, "loss": 1.8928, "theoretical_loss": 4.693504011803159, "tokens_seen": 118030336 }, { "epoch": 0.04, "learning_rate": 0.0007791695085255768, "loss": 1.8751, "theoretical_loss": 4.6931533757987065, "tokens_seen": 118095872 }, { "epoch": 0.04, "learning_rate": 0.0007791534603811435, "loss": 1.9013, "theoretical_loss": 4.692802988770258, "tokens_seen": 118161408 }, { "epoch": 0.04, "learning_rate": 0.0007791374122367102, "loss": 1.8634, "theoretical_loss": 4.692452850403081, "tokens_seen": 118226944 }, { "epoch": 0.04, "learning_rate": 0.0007791213640922769, "loss": 2.0152, "theoretical_loss": 4.692102960383014, "tokens_seen": 118292480 }, { "epoch": 0.04, "learning_rate": 0.0007791053159478436, "loss": 1.7348, "theoretical_loss": 4.691753318396467, "tokens_seen": 118358016 }, { "epoch": 0.04, "learning_rate": 0.0007790892678034103, "loss": 1.8863, "theoretical_loss": 4.691403924130418, "tokens_seen": 118423552 }, { "epoch": 0.04, "learning_rate": 0.000779073219658977, "loss": 1.9038, "theoretical_loss": 4.691054777272415, "tokens_seen": 118489088 }, { "epoch": 0.04, "learning_rate": 0.0007790571715145437, "loss": 1.8282, "theoretical_loss": 4.69070587751057, "tokens_seen": 118554624 }, { "epoch": 0.04, "learning_rate": 0.0007790411233701104, "loss": 1.9085, "theoretical_loss": 4.690357224533564, "tokens_seen": 118620160 }, { "epoch": 0.04, "learning_rate": 0.0007790250752256771, "loss": 1.8251, "theoretical_loss": 4.690008818030638, "tokens_seen": 118685696 }, { "epoch": 0.04, "learning_rate": 0.0007790090270812438, "loss": 1.6978, "theoretical_loss": 4.689660657691598, "tokens_seen": 118751232 }, { "epoch": 0.04, "learning_rate": 0.0007789929789368105, "loss": 1.9915, "theoretical_loss": 4.689312743206811, "tokens_seen": 118816768 }, { "epoch": 0.04, "learning_rate": 0.0007789769307923772, "loss": 1.9805, "theoretical_loss": 4.688965074267203, "tokens_seen": 118882304 }, { "epoch": 0.04, "learning_rate": 0.0007789608826479439, "loss": 1.8472, "theoretical_loss": 4.688617650564259, "tokens_seen": 118947840 }, { "epoch": 0.04, "learning_rate": 0.0007789448345035106, "loss": 1.9364, "theoretical_loss": 4.688270471790023, "tokens_seen": 119013376 }, { "epoch": 0.04, "learning_rate": 0.0007789287863590773, "loss": 1.8973, "theoretical_loss": 4.687923537637094, "tokens_seen": 119078912 }, { "epoch": 0.04, "learning_rate": 0.000778912738214644, "loss": 1.8099, "theoretical_loss": 4.687576847798622, "tokens_seen": 119144448 }, { "epoch": 0.04, "learning_rate": 0.0007788966900702107, "loss": 1.8415, "theoretical_loss": 4.687230401968319, "tokens_seen": 119209984 }, { "epoch": 0.04, "learning_rate": 0.0007788806419257774, "loss": 1.9467, "theoretical_loss": 4.68688419984044, "tokens_seen": 119275520 }, { "epoch": 0.04, "learning_rate": 0.0007788645937813441, "loss": 1.8803, "theoretical_loss": 4.686538241109796, "tokens_seen": 119341056 }, { "epoch": 0.04, "learning_rate": 0.0007788485456369108, "loss": 1.8097, "theoretical_loss": 4.686192525471746, "tokens_seen": 119406592 }, { "epoch": 0.04, "learning_rate": 0.0007788324974924775, "loss": 1.8324, "theoretical_loss": 4.685847052622199, "tokens_seen": 119472128 }, { "epoch": 0.04, "learning_rate": 0.0007788164493480442, "loss": 1.8055, "theoretical_loss": 4.685501822257608, "tokens_seen": 119537664 }, { "epoch": 0.04, "learning_rate": 0.0007788004012036109, "loss": 1.7789, "theoretical_loss": 4.685156834074972, "tokens_seen": 119603200 }, { "epoch": 0.04, "learning_rate": 0.0007787843530591776, "loss": 1.763, "theoretical_loss": 4.684812087771839, "tokens_seen": 119668736 }, { "epoch": 0.04, "learning_rate": 0.0007787683049147443, "loss": 1.8904, "theoretical_loss": 4.684467583046292, "tokens_seen": 119734272 }, { "epoch": 0.04, "learning_rate": 0.000778752256770311, "loss": 1.84, "theoretical_loss": 4.6841233195969645, "tokens_seen": 119799808 }, { "epoch": 0.04, "learning_rate": 0.0007787362086258777, "loss": 1.9018, "theoretical_loss": 4.683779297123024, "tokens_seen": 119865344 }, { "epoch": 0.04, "learning_rate": 0.0007787201604814444, "loss": 1.804, "theoretical_loss": 4.683435515324181, "tokens_seen": 119930880 }, { "epoch": 0.04, "learning_rate": 0.0007787041123370111, "loss": 1.9663, "theoretical_loss": 4.683091973900682, "tokens_seen": 119996416 }, { "epoch": 0.04, "learning_rate": 0.0007786880641925778, "loss": 1.849, "theoretical_loss": 4.682748672553313, "tokens_seen": 120061952 }, { "epoch": 0.04, "learning_rate": 0.0007786720160481445, "loss": 1.8176, "theoretical_loss": 4.68240561098339, "tokens_seen": 120127488 }, { "epoch": 0.04, "learning_rate": 0.0007786559679037112, "loss": 1.7518, "theoretical_loss": 4.68206278889277, "tokens_seen": 120193024 }, { "epoch": 0.04, "learning_rate": 0.0007786399197592779, "loss": 1.8349, "theoretical_loss": 4.681720205983839, "tokens_seen": 120258560 }, { "epoch": 0.04, "learning_rate": 0.0007786238716148446, "loss": 1.8489, "theoretical_loss": 4.681377861959516, "tokens_seen": 120324096 }, { "epoch": 0.04, "learning_rate": 0.0007786078234704113, "loss": 1.8931, "theoretical_loss": 4.681035756523249, "tokens_seen": 120389632 }, { "epoch": 0.04, "learning_rate": 0.0007785917753259779, "loss": 1.9067, "theoretical_loss": 4.680693889379018, "tokens_seen": 120455168 }, { "epoch": 0.04, "learning_rate": 0.0007785757271815447, "loss": 1.855, "theoretical_loss": 4.680352260231329, "tokens_seen": 120520704 }, { "epoch": 0.04, "learning_rate": 0.0007785596790371113, "loss": 1.8698, "theoretical_loss": 4.680010868785216, "tokens_seen": 120586240 }, { "epoch": 0.04, "learning_rate": 0.0007785436308926781, "loss": 1.83, "theoretical_loss": 4.679669714746238, "tokens_seen": 120651776 }, { "epoch": 0.04, "learning_rate": 0.0007785275827482447, "loss": 1.9867, "theoretical_loss": 4.679328797820478, "tokens_seen": 120717312 }, { "epoch": 0.04, "learning_rate": 0.0007785115346038115, "loss": 1.944, "theoretical_loss": 4.678988117714544, "tokens_seen": 120782848 }, { "epoch": 0.04, "learning_rate": 0.0007784954864593781, "loss": 1.7626, "theoretical_loss": 4.678647674135563, "tokens_seen": 120848384 }, { "epoch": 0.04, "learning_rate": 0.0007784794383149449, "loss": 1.8657, "theoretical_loss": 4.6783074667911855, "tokens_seen": 120913920 }, { "epoch": 0.04, "learning_rate": 0.0007784633901705115, "loss": 1.8642, "theoretical_loss": 4.677967495389581, "tokens_seen": 120979456 }, { "epoch": 0.04, "learning_rate": 0.0007784473420260782, "loss": 1.7655, "theoretical_loss": 4.677627759639435, "tokens_seen": 121044992 }, { "epoch": 0.04, "learning_rate": 0.0007784312938816449, "loss": 1.8806, "theoretical_loss": 4.677288259249954, "tokens_seen": 121110528 }, { "epoch": 0.04, "learning_rate": 0.0007784152457372116, "loss": 1.8441, "theoretical_loss": 4.676948993930857, "tokens_seen": 121176064 }, { "epoch": 0.04, "objective/train/docs_used": 47440, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.9147517681121826, "objective/train/theoretical_loss": 4.67660996339238, "objective/train/tokens_used": 141701600, "theoretical_loss": 4.67660996339238, "tokens_seen": 121241600 }, { "epoch": 0.04, "learning_rate": 0.0007783991975927783, "loss": 1.9058, "theoretical_loss": 4.67660996339238, "tokens_seen": 121241600 }, { "epoch": 0.04, "learning_rate": 0.000778383149448345, "loss": 1.9118, "theoretical_loss": 4.676271167345272, "tokens_seen": 121307136 }, { "epoch": 0.04, "learning_rate": 0.0007783671013039117, "loss": 1.9903, "theoretical_loss": 4.675932605500794, "tokens_seen": 121372672 }, { "epoch": 0.04, "learning_rate": 0.0007783510531594785, "loss": 1.846, "theoretical_loss": 4.675594277570718, "tokens_seen": 121438208 }, { "epoch": 0.04, "learning_rate": 0.0007783350050150451, "loss": 1.7874, "theoretical_loss": 4.675256183267327, "tokens_seen": 121503744 }, { "epoch": 0.04, "learning_rate": 0.0007783189568706119, "loss": 1.8911, "theoretical_loss": 4.674918322303411, "tokens_seen": 121569280 }, { "epoch": 0.04, "learning_rate": 0.0007783029087261785, "loss": 1.7944, "theoretical_loss": 4.6745806943922705, "tokens_seen": 121634816 }, { "epoch": 0.04, "learning_rate": 0.0007782868605817453, "loss": 1.7367, "theoretical_loss": 4.67424329924771, "tokens_seen": 121700352 }, { "epoch": 0.04, "learning_rate": 0.0007782708124373119, "loss": 1.8909, "theoretical_loss": 4.673906136584041, "tokens_seen": 121765888 }, { "epoch": 0.04, "learning_rate": 0.0007782547642928787, "loss": 1.9456, "theoretical_loss": 4.673569206116078, "tokens_seen": 121831424 }, { "epoch": 0.04, "learning_rate": 0.0007782387161484453, "loss": 1.8463, "theoretical_loss": 4.6732325075591366, "tokens_seen": 121896960 }, { "epoch": 0.04, "learning_rate": 0.0007782226680040121, "loss": 1.9333, "theoretical_loss": 4.67289604062904, "tokens_seen": 121962496 }, { "epoch": 0.04, "learning_rate": 0.0007782066198595787, "loss": 1.7753, "theoretical_loss": 4.672559805042107, "tokens_seen": 122028032 }, { "epoch": 0.04, "learning_rate": 0.0007781905717151455, "loss": 1.8157, "theoretical_loss": 4.672223800515159, "tokens_seen": 122093568 }, { "epoch": 0.04, "learning_rate": 0.0007781745235707121, "loss": 1.8541, "theoretical_loss": 4.671888026765512, "tokens_seen": 122159104 }, { "epoch": 0.04, "learning_rate": 0.0007781584754262789, "loss": 1.7951, "theoretical_loss": 4.6715524835109825, "tokens_seen": 122224640 }, { "epoch": 0.04, "learning_rate": 0.0007781424272818455, "loss": 1.8092, "theoretical_loss": 4.671217170469884, "tokens_seen": 122290176 }, { "epoch": 0.04, "learning_rate": 0.0007781263791374123, "loss": 1.7207, "theoretical_loss": 4.67088208736102, "tokens_seen": 122355712 }, { "epoch": 0.04, "learning_rate": 0.0007781103309929789, "loss": 1.9078, "theoretical_loss": 4.670547233903694, "tokens_seen": 122421248 }, { "epoch": 0.04, "learning_rate": 0.0007780942828485457, "loss": 1.8607, "theoretical_loss": 4.670212609817698, "tokens_seen": 122486784 }, { "epoch": 0.04, "learning_rate": 0.0007780782347041123, "loss": 1.8305, "theoretical_loss": 4.669878214823316, "tokens_seen": 122552320 }, { "epoch": 0.04, "learning_rate": 0.0007780621865596791, "loss": 1.9033, "theoretical_loss": 4.669544048641325, "tokens_seen": 122617856 }, { "epoch": 0.04, "learning_rate": 0.0007780461384152457, "loss": 1.9115, "theoretical_loss": 4.669210110992989, "tokens_seen": 122683392 }, { "epoch": 0.04, "learning_rate": 0.0007780300902708125, "loss": 1.9634, "theoretical_loss": 4.668876401600059, "tokens_seen": 122748928 }, { "epoch": 0.04, "learning_rate": 0.0007780140421263791, "loss": 1.7661, "theoretical_loss": 4.668542920184779, "tokens_seen": 122814464 }, { "epoch": 0.04, "learning_rate": 0.0007779979939819459, "loss": 1.8726, "theoretical_loss": 4.6682096664698705, "tokens_seen": 122880000 }, { "epoch": 0.04, "learning_rate": 0.0007779819458375126, "loss": 1.9475, "theoretical_loss": 4.667876640178546, "tokens_seen": 122945536 }, { "epoch": 0.04, "learning_rate": 0.0007779658976930793, "loss": 1.9562, "theoretical_loss": 4.667543841034502, "tokens_seen": 123011072 }, { "epoch": 0.04, "learning_rate": 0.000777949849548646, "loss": 1.9686, "theoretical_loss": 4.667211268761914, "tokens_seen": 123076608 }, { "epoch": 0.04, "learning_rate": 0.0007779338014042127, "loss": 1.8776, "theoretical_loss": 4.66687892308544, "tokens_seen": 123142144 }, { "epoch": 0.04, "learning_rate": 0.0007779177532597794, "loss": 1.9679, "theoretical_loss": 4.6665468037302205, "tokens_seen": 123207680 }, { "epoch": 0.04, "learning_rate": 0.0007779017051153461, "loss": 1.804, "theoretical_loss": 4.666214910421875, "tokens_seen": 123273216 }, { "epoch": 0.04, "learning_rate": 0.0007778856569709128, "loss": 1.8628, "theoretical_loss": 4.665883242886498, "tokens_seen": 123338752 }, { "epoch": 0.04, "learning_rate": 0.0007778696088264795, "loss": 1.8482, "theoretical_loss": 4.6655518008506665, "tokens_seen": 123404288 }, { "epoch": 0.04, "learning_rate": 0.0007778535606820462, "loss": 1.8696, "theoretical_loss": 4.665220584041428, "tokens_seen": 123469824 }, { "epoch": 0.04, "learning_rate": 0.0007778375125376129, "loss": 1.9316, "theoretical_loss": 4.664889592186309, "tokens_seen": 123535360 }, { "epoch": 0.04, "learning_rate": 0.0007778214643931796, "loss": 1.9424, "theoretical_loss": 4.6645588250133105, "tokens_seen": 123600896 }, { "epoch": 0.04, "learning_rate": 0.0007778054162487463, "loss": 1.9075, "theoretical_loss": 4.664228282250902, "tokens_seen": 123666432 }, { "epoch": 0.04, "learning_rate": 0.000777789368104313, "loss": 1.8398, "theoretical_loss": 4.663897963628029, "tokens_seen": 123731968 }, { "epoch": 0.04, "learning_rate": 0.0007777733199598797, "loss": 1.8985, "theoretical_loss": 4.6635678688741065, "tokens_seen": 123797504 }, { "epoch": 0.04, "learning_rate": 0.0007777572718154464, "loss": 1.9671, "theoretical_loss": 4.663237997719017, "tokens_seen": 123863040 }, { "epoch": 0.04, "learning_rate": 0.0007777412236710131, "loss": 1.8071, "theoretical_loss": 4.662908349893116, "tokens_seen": 123928576 }, { "epoch": 0.04, "learning_rate": 0.0007777251755265798, "loss": 1.9107, "theoretical_loss": 4.662578925127223, "tokens_seen": 123994112 }, { "epoch": 0.04, "learning_rate": 0.0007777091273821465, "loss": 1.7927, "theoretical_loss": 4.662249723152624, "tokens_seen": 124059648 }, { "epoch": 0.04, "learning_rate": 0.0007776930792377132, "loss": 1.9306, "theoretical_loss": 4.661920743701073, "tokens_seen": 124125184 }, { "epoch": 0.04, "learning_rate": 0.0007776770310932799, "loss": 2.021, "theoretical_loss": 4.661591986504786, "tokens_seen": 124190720 }, { "epoch": 0.04, "learning_rate": 0.0007776609829488466, "loss": 1.9438, "theoretical_loss": 4.661263451296444, "tokens_seen": 124256256 }, { "epoch": 0.04, "learning_rate": 0.0007776449348044133, "loss": 1.8106, "theoretical_loss": 4.660935137809188, "tokens_seen": 124321792 }, { "epoch": 0.04, "learning_rate": 0.00077762888665998, "loss": 1.8744, "theoretical_loss": 4.660607045776624, "tokens_seen": 124387328 }, { "epoch": 0.04, "learning_rate": 0.0007776128385155467, "loss": 1.7867, "theoretical_loss": 4.660279174932814, "tokens_seen": 124452864 }, { "epoch": 0.04, "objective/train/docs_used": 48338, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7055823802947998, "objective/train/theoretical_loss": 4.659951525012284, "objective/train/tokens_used": 144978400, "theoretical_loss": 4.659951525012284, "tokens_seen": 124518400 }, { "epoch": 0.04, "learning_rate": 0.0007775967903711134, "loss": 1.8029, "theoretical_loss": 4.659951525012284, "tokens_seen": 124518400 }, { "epoch": 0.04, "learning_rate": 0.0007775807422266801, "loss": 1.8085, "theoretical_loss": 4.6596240957500115, "tokens_seen": 124583936 }, { "epoch": 0.04, "learning_rate": 0.0007775646940822468, "loss": 1.8204, "theoretical_loss": 4.659296886881438, "tokens_seen": 124649472 }, { "epoch": 0.04, "learning_rate": 0.0007775486459378135, "loss": 1.8372, "theoretical_loss": 4.658969898142456, "tokens_seen": 124715008 }, { "epoch": 0.04, "learning_rate": 0.0007775325977933802, "loss": 1.7415, "theoretical_loss": 4.658643129269416, "tokens_seen": 124780544 }, { "epoch": 0.04, "learning_rate": 0.0007775165496489469, "loss": 1.7994, "theoretical_loss": 4.658316579999122, "tokens_seen": 124846080 }, { "epoch": 0.04, "learning_rate": 0.0007775005015045136, "loss": 1.8632, "theoretical_loss": 4.65799025006883, "tokens_seen": 124911616 }, { "epoch": 0.04, "learning_rate": 0.0007774844533600803, "loss": 1.8766, "theoretical_loss": 4.65766413921625, "tokens_seen": 124977152 }, { "epoch": 0.04, "learning_rate": 0.000777468405215647, "loss": 1.8277, "theoretical_loss": 4.657338247179538, "tokens_seen": 125042688 }, { "epoch": 0.04, "learning_rate": 0.0007774523570712137, "loss": 1.8551, "theoretical_loss": 4.657012573697307, "tokens_seen": 125108224 }, { "epoch": 0.04, "learning_rate": 0.0007774363089267804, "loss": 1.8749, "theoretical_loss": 4.656687118508616, "tokens_seen": 125173760 }, { "epoch": 0.04, "learning_rate": 0.0007774202607823471, "loss": 1.7936, "theoretical_loss": 4.656361881352968, "tokens_seen": 125239296 }, { "epoch": 0.04, "learning_rate": 0.0007774042126379138, "loss": 1.7392, "theoretical_loss": 4.656036861970319, "tokens_seen": 125304832 }, { "epoch": 0.04, "learning_rate": 0.0007773881644934805, "loss": 1.8527, "theoretical_loss": 4.655712060101068, "tokens_seen": 125370368 }, { "epoch": 0.04, "learning_rate": 0.0007773721163490472, "loss": 1.854, "theoretical_loss": 4.65538747548606, "tokens_seen": 125435904 }, { "epoch": 0.04, "learning_rate": 0.0007773560682046139, "loss": 1.8442, "theoretical_loss": 4.655063107866582, "tokens_seen": 125501440 }, { "epoch": 0.04, "learning_rate": 0.0007773400200601806, "loss": 2.0244, "theoretical_loss": 4.654738956984366, "tokens_seen": 125566976 }, { "epoch": 0.04, "learning_rate": 0.0007773239719157473, "loss": 1.7517, "theoretical_loss": 4.654415022581585, "tokens_seen": 125632512 }, { "epoch": 0.04, "learning_rate": 0.000777307923771314, "loss": 1.8149, "theoretical_loss": 4.654091304400853, "tokens_seen": 125698048 }, { "epoch": 0.04, "learning_rate": 0.0007772918756268807, "loss": 1.9215, "theoretical_loss": 4.653767802185225, "tokens_seen": 125763584 }, { "epoch": 0.04, "learning_rate": 0.0007772758274824474, "loss": 1.8353, "theoretical_loss": 4.653444515678195, "tokens_seen": 125829120 }, { "epoch": 0.04, "learning_rate": 0.0007772597793380141, "loss": 1.8363, "theoretical_loss": 4.653121444623693, "tokens_seen": 125894656 }, { "epoch": 0.04, "learning_rate": 0.0007772437311935808, "loss": 1.7447, "theoretical_loss": 4.652798588766088, "tokens_seen": 125960192 }, { "epoch": 0.04, "learning_rate": 0.0007772276830491475, "loss": 1.7671, "theoretical_loss": 4.652475947850185, "tokens_seen": 126025728 }, { "epoch": 0.04, "learning_rate": 0.0007772116349047142, "loss": 1.753, "theoretical_loss": 4.652153521621223, "tokens_seen": 126091264 }, { "epoch": 0.04, "learning_rate": 0.0007771955867602808, "loss": 1.9541, "theoretical_loss": 4.651831309824875, "tokens_seen": 126156800 }, { "epoch": 0.04, "learning_rate": 0.0007771795386158476, "loss": 1.7825, "theoretical_loss": 4.65150931220725, "tokens_seen": 126222336 }, { "epoch": 0.04, "learning_rate": 0.0007771634904714142, "loss": 1.9339, "theoretical_loss": 4.651187528514887, "tokens_seen": 126287872 }, { "epoch": 0.04, "learning_rate": 0.000777147442326981, "loss": 1.8252, "theoretical_loss": 4.650865958494755, "tokens_seen": 126353408 }, { "epoch": 0.04, "learning_rate": 0.0007771313941825476, "loss": 1.8909, "theoretical_loss": 4.650544601894257, "tokens_seen": 126418944 }, { "epoch": 0.04, "learning_rate": 0.0007771153460381144, "loss": 1.9024, "theoretical_loss": 4.650223458461221, "tokens_seen": 126484480 }, { "epoch": 0.04, "learning_rate": 0.000777099297893681, "loss": 1.862, "theoretical_loss": 4.649902527943908, "tokens_seen": 126550016 }, { "epoch": 0.04, "learning_rate": 0.0007770832497492478, "loss": 1.8003, "theoretical_loss": 4.649581810091003, "tokens_seen": 126615552 }, { "epoch": 0.04, "learning_rate": 0.0007770672016048144, "loss": 2.0104, "theoretical_loss": 4.649261304651619, "tokens_seen": 126681088 }, { "epoch": 0.04, "learning_rate": 0.0007770511534603812, "loss": 1.7893, "theoretical_loss": 4.6489410113752925, "tokens_seen": 126746624 }, { "epoch": 0.04, "learning_rate": 0.0007770351053159478, "loss": 1.814, "theoretical_loss": 4.6486209300119885, "tokens_seen": 126812160 }, { "epoch": 0.04, "learning_rate": 0.0007770190571715146, "loss": 1.9046, "theoretical_loss": 4.648301060312093, "tokens_seen": 126877696 }, { "epoch": 0.04, "learning_rate": 0.0007770030090270812, "loss": 1.8817, "theoretical_loss": 4.647981402026417, "tokens_seen": 126943232 }, { "epoch": 0.04, "learning_rate": 0.000776986960882648, "loss": 1.8367, "theoretical_loss": 4.647661954906188, "tokens_seen": 127008768 }, { "epoch": 0.04, "learning_rate": 0.0007769709127382146, "loss": 1.7926, "theoretical_loss": 4.647342718703062, "tokens_seen": 127074304 }, { "epoch": 0.04, "learning_rate": 0.0007769548645937814, "loss": 1.9298, "theoretical_loss": 4.647023693169107, "tokens_seen": 127139840 }, { "epoch": 0.04, "learning_rate": 0.000776938816449348, "loss": 1.7827, "theoretical_loss": 4.646704878056816, "tokens_seen": 127205376 }, { "epoch": 0.04, "learning_rate": 0.0007769227683049148, "loss": 1.7387, "theoretical_loss": 4.6463862731191, "tokens_seen": 127270912 }, { "epoch": 0.04, "learning_rate": 0.0007769067201604814, "loss": 1.91, "theoretical_loss": 4.6460678781092835, "tokens_seen": 127336448 }, { "epoch": 0.04, "learning_rate": 0.0007768906720160482, "loss": 1.993, "theoretical_loss": 4.6457496927811075, "tokens_seen": 127401984 }, { "epoch": 0.04, "learning_rate": 0.0007768746238716148, "loss": 1.6914, "theoretical_loss": 4.645431716888734, "tokens_seen": 127467520 }, { "epoch": 0.04, "learning_rate": 0.0007768585757271816, "loss": 1.8668, "theoretical_loss": 4.645113950186732, "tokens_seen": 127533056 }, { "epoch": 0.04, "learning_rate": 0.0007768425275827482, "loss": 1.7944, "theoretical_loss": 4.64479639243009, "tokens_seen": 127598592 }, { "epoch": 0.04, "learning_rate": 0.000776826479438315, "loss": 1.7889, "theoretical_loss": 4.644479043374204, "tokens_seen": 127664128 }, { "epoch": 0.04, "learning_rate": 0.0007768104312938816, "loss": 1.8859, "theoretical_loss": 4.644161902774886, "tokens_seen": 127729664 }, { "epoch": 0.04, "objective/train/docs_used": 49581, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.94822096824646, "objective/train/theoretical_loss": 4.643844970388358, "objective/train/tokens_used": 148255200, "theoretical_loss": 4.643844970388358, "tokens_seen": 127795200 }, { "epoch": 0.04, "learning_rate": 0.0007767943831494484, "loss": 1.9133, "theoretical_loss": 4.643844970388358, "tokens_seen": 127795200 }, { "epoch": 0.04, "learning_rate": 0.000776778335005015, "loss": 1.8609, "theoretical_loss": 4.6435282459712495, "tokens_seen": 127860736 }, { "epoch": 0.04, "learning_rate": 0.0007767622868605818, "loss": 1.8257, "theoretical_loss": 4.643211729280601, "tokens_seen": 127926272 }, { "epoch": 0.04, "learning_rate": 0.0007767462387161484, "loss": 1.7747, "theoretical_loss": 4.64289542007386, "tokens_seen": 127991808 }, { "epoch": 0.04, "learning_rate": 0.0007767301905717152, "loss": 1.8617, "theoretical_loss": 4.642579318108883, "tokens_seen": 128057344 }, { "epoch": 0.04, "learning_rate": 0.0007767141424272818, "loss": 1.8137, "theoretical_loss": 4.642263423143932, "tokens_seen": 128122880 }, { "epoch": 0.04, "learning_rate": 0.0007766980942828486, "loss": 1.8099, "theoretical_loss": 4.641947734937673, "tokens_seen": 128188416 }, { "epoch": 0.04, "learning_rate": 0.0007766820461384152, "loss": 1.8716, "theoretical_loss": 4.641632253249179, "tokens_seen": 128253952 }, { "epoch": 0.04, "learning_rate": 0.000776665997993982, "loss": 2.0769, "theoretical_loss": 4.641316977837924, "tokens_seen": 128319488 }, { "epoch": 0.04, "learning_rate": 0.0007766499498495486, "loss": 1.89, "theoretical_loss": 4.641001908463787, "tokens_seen": 128385024 }, { "epoch": 0.04, "learning_rate": 0.0007766339017051154, "loss": 1.8734, "theoretical_loss": 4.6406870448870485, "tokens_seen": 128450560 }, { "epoch": 0.04, "learning_rate": 0.000776617853560682, "loss": 1.8353, "theoretical_loss": 4.64037238686839, "tokens_seen": 128516096 }, { "epoch": 0.04, "learning_rate": 0.0007766018054162488, "loss": 1.8572, "theoretical_loss": 4.640057934168891, "tokens_seen": 128581632 }, { "epoch": 0.04, "learning_rate": 0.0007765857572718155, "loss": 1.7848, "theoretical_loss": 4.639743686550034, "tokens_seen": 128647168 }, { "epoch": 0.04, "learning_rate": 0.0007765697091273822, "loss": 1.8396, "theoretical_loss": 4.639429643773697, "tokens_seen": 128712704 }, { "epoch": 0.04, "learning_rate": 0.0007765536609829489, "loss": 1.8095, "theoretical_loss": 4.639115805602156, "tokens_seen": 128778240 }, { "epoch": 0.04, "learning_rate": 0.0007765376128385156, "loss": 1.8191, "theoretical_loss": 4.638802171798086, "tokens_seen": 128843776 }, { "epoch": 0.04, "learning_rate": 0.0007765215646940823, "loss": 1.9008, "theoretical_loss": 4.638488742124554, "tokens_seen": 128909312 }, { "epoch": 0.04, "learning_rate": 0.000776505516549649, "loss": 1.8786, "theoretical_loss": 4.638175516345027, "tokens_seen": 128974848 }, { "epoch": 0.04, "learning_rate": 0.0007764894684052157, "loss": 1.8323, "theoretical_loss": 4.637862494223359, "tokens_seen": 129040384 }, { "epoch": 0.04, "learning_rate": 0.0007764734202607824, "loss": 1.7773, "theoretical_loss": 4.637549675523806, "tokens_seen": 129105920 }, { "epoch": 0.04, "learning_rate": 0.0007764573721163491, "loss": 1.8804, "theoretical_loss": 4.637237060011007, "tokens_seen": 129171456 }, { "epoch": 0.04, "learning_rate": 0.0007764413239719158, "loss": 1.847, "theoretical_loss": 4.636924647450002, "tokens_seen": 129236992 }, { "epoch": 0.04, "learning_rate": 0.0007764252758274825, "loss": 1.8726, "theoretical_loss": 4.636612437606212, "tokens_seen": 129302528 }, { "epoch": 0.04, "learning_rate": 0.0007764092276830492, "loss": 1.8728, "theoretical_loss": 4.636300430245457, "tokens_seen": 129368064 }, { "epoch": 0.04, "learning_rate": 0.0007763931795386159, "loss": 1.7916, "theoretical_loss": 4.63598862513394, "tokens_seen": 129433600 }, { "epoch": 0.04, "learning_rate": 0.0007763771313941826, "loss": 1.8813, "theoretical_loss": 4.635677022038254, "tokens_seen": 129499136 }, { "epoch": 0.04, "learning_rate": 0.0007763610832497493, "loss": 1.8075, "theoretical_loss": 4.63536562072538, "tokens_seen": 129564672 }, { "epoch": 0.04, "learning_rate": 0.000776345035105316, "loss": 1.8598, "theoretical_loss": 4.635054420962684, "tokens_seen": 129630208 }, { "epoch": 0.04, "learning_rate": 0.0007763289869608827, "loss": 1.8148, "theoretical_loss": 4.634743422517918, "tokens_seen": 129695744 }, { "epoch": 0.04, "learning_rate": 0.0007763129388164494, "loss": 1.7992, "theoretical_loss": 4.634432625159221, "tokens_seen": 129761280 }, { "epoch": 0.04, "learning_rate": 0.0007762968906720161, "loss": 1.8888, "theoretical_loss": 4.634122028655112, "tokens_seen": 129826816 }, { "epoch": 0.04, "learning_rate": 0.0007762808425275828, "loss": 1.7781, "theoretical_loss": 4.633811632774496, "tokens_seen": 129892352 }, { "epoch": 0.04, "learning_rate": 0.0007762647943831495, "loss": 1.8929, "theoretical_loss": 4.633501437286659, "tokens_seen": 129957888 }, { "epoch": 0.04, "learning_rate": 0.0007762487462387162, "loss": 1.9065, "theoretical_loss": 4.633191441961272, "tokens_seen": 130023424 }, { "epoch": 0.04, "learning_rate": 0.0007762326980942829, "loss": 1.7713, "theoretical_loss": 4.63288164656838, "tokens_seen": 130088960 }, { "epoch": 0.04, "learning_rate": 0.0007762166499498496, "loss": 1.7753, "theoretical_loss": 4.632572050878415, "tokens_seen": 130154496 }, { "epoch": 0.04, "learning_rate": 0.0007762006018054163, "loss": 1.8997, "theoretical_loss": 4.632262654662181, "tokens_seen": 130220032 }, { "epoch": 0.04, "learning_rate": 0.000776184553660983, "loss": 1.8561, "theoretical_loss": 4.631953457690868, "tokens_seen": 130285568 }, { "epoch": 0.04, "learning_rate": 0.0007761685055165497, "loss": 1.9013, "theoretical_loss": 4.631644459736036, "tokens_seen": 130351104 }, { "epoch": 0.04, "learning_rate": 0.0007761524573721164, "loss": 1.9393, "theoretical_loss": 4.631335660569626, "tokens_seen": 130416640 }, { "epoch": 0.04, "learning_rate": 0.0007761364092276831, "loss": 1.9441, "theoretical_loss": 4.631027059963953, "tokens_seen": 130482176 }, { "epoch": 0.04, "learning_rate": 0.0007761203610832498, "loss": 1.8176, "theoretical_loss": 4.630718657691709, "tokens_seen": 130547712 }, { "epoch": 0.04, "learning_rate": 0.0007761043129388165, "loss": 1.7216, "theoretical_loss": 4.630410453525958, "tokens_seen": 130613248 }, { "epoch": 0.04, "learning_rate": 0.0007760882647943832, "loss": 1.766, "theoretical_loss": 4.630102447240138, "tokens_seen": 130678784 }, { "epoch": 0.04, "learning_rate": 0.0007760722166499499, "loss": 1.8036, "theoretical_loss": 4.629794638608059, "tokens_seen": 130744320 }, { "epoch": 0.04, "learning_rate": 0.0007760561685055166, "loss": 1.851, "theoretical_loss": 4.629487027403905, "tokens_seen": 130809856 }, { "epoch": 0.04, "learning_rate": 0.0007760401203610833, "loss": 1.8026, "theoretical_loss": 4.629179613402228, "tokens_seen": 130875392 }, { "epoch": 0.04, "learning_rate": 0.00077602407221665, "loss": 1.8168, "theoretical_loss": 4.628872396377953, "tokens_seen": 130940928 }, { "epoch": 0.04, "learning_rate": 0.0007760080240722167, "loss": 1.872, "theoretical_loss": 4.628565376106371, "tokens_seen": 131006464 }, { "epoch": 0.04, "objective/train/docs_used": 50603, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7769169807434082, "objective/train/theoretical_loss": 4.628258552363146, "objective/train/tokens_used": 151532000, "theoretical_loss": 4.628258552363146, "tokens_seen": 131072000 }, { "epoch": 0.04, "learning_rate": 0.0007759919759277834, "loss": 1.8162, "theoretical_loss": 4.628258552363146, "tokens_seen": 131072000 }, { "epoch": 0.04, "learning_rate": 0.0007759759277833502, "loss": 1.8492, "theoretical_loss": 4.627951924924305, "tokens_seen": 131137536 }, { "epoch": 0.04, "learning_rate": 0.0007759598796389168, "loss": 1.8724, "theoretical_loss": 4.627645493566245, "tokens_seen": 131203072 }, { "epoch": 0.04, "learning_rate": 0.0007759438314944836, "loss": 1.8344, "theoretical_loss": 4.62733925806573, "tokens_seen": 131268608 }, { "epoch": 0.04, "learning_rate": 0.0007759277833500502, "loss": 1.8899, "theoretical_loss": 4.627033218199886, "tokens_seen": 131334144 }, { "epoch": 0.04, "learning_rate": 0.0007759117352056169, "loss": 1.8562, "theoretical_loss": 4.626727373746206, "tokens_seen": 131399680 }, { "epoch": 0.04, "learning_rate": 0.0007758956870611836, "loss": 1.8066, "theoretical_loss": 4.626421724482549, "tokens_seen": 131465216 }, { "epoch": 0.04, "learning_rate": 0.0007758796389167503, "loss": 1.8461, "theoretical_loss": 4.626116270187131, "tokens_seen": 131530752 }, { "epoch": 0.04, "learning_rate": 0.000775863590772317, "loss": 1.736, "theoretical_loss": 4.625811010638538, "tokens_seen": 131596288 }, { "epoch": 0.04, "learning_rate": 0.0007758475426278837, "loss": 1.8001, "theoretical_loss": 4.62550594561571, "tokens_seen": 131661824 }, { "epoch": 0.04, "learning_rate": 0.0007758314944834504, "loss": 1.823, "theoretical_loss": 4.625201074897952, "tokens_seen": 131727360 }, { "epoch": 0.04, "learning_rate": 0.0007758154463390171, "loss": 1.9525, "theoretical_loss": 4.62489639826493, "tokens_seen": 131792896 }, { "epoch": 0.04, "learning_rate": 0.0007757993981945837, "loss": 1.8137, "theoretical_loss": 4.6245919154966675, "tokens_seen": 131858432 }, { "epoch": 0.04, "learning_rate": 0.0007757833500501505, "loss": 1.8273, "theoretical_loss": 4.624287626373545, "tokens_seen": 131923968 }, { "epoch": 0.04, "learning_rate": 0.0007757673019057171, "loss": 1.7471, "theoretical_loss": 4.623983530676304, "tokens_seen": 131989504 }, { "epoch": 0.04, "learning_rate": 0.0007757512537612839, "loss": 1.8519, "theoretical_loss": 4.62367962818604, "tokens_seen": 132055040 }, { "epoch": 0.04, "learning_rate": 0.0007757352056168505, "loss": 1.944, "theoretical_loss": 4.623375918684207, "tokens_seen": 132120576 }, { "epoch": 0.04, "learning_rate": 0.0007757191574724173, "loss": 1.8725, "theoretical_loss": 4.623072401952614, "tokens_seen": 132186112 }, { "epoch": 0.04, "learning_rate": 0.0007757031093279839, "loss": 1.8953, "theoretical_loss": 4.622769077773423, "tokens_seen": 132251648 }, { "epoch": 0.04, "learning_rate": 0.0007756870611835507, "loss": 1.879, "theoretical_loss": 4.622465945929152, "tokens_seen": 132317184 }, { "epoch": 0.04, "learning_rate": 0.0007756710130391173, "loss": 1.7999, "theoretical_loss": 4.622163006202671, "tokens_seen": 132382720 }, { "epoch": 0.04, "learning_rate": 0.0007756549648946841, "loss": 1.7466, "theoretical_loss": 4.621860258377203, "tokens_seen": 132448256 }, { "epoch": 0.04, "learning_rate": 0.0007756389167502507, "loss": 1.8657, "theoretical_loss": 4.621557702236323, "tokens_seen": 132513792 }, { "epoch": 0.04, "learning_rate": 0.0007756228686058175, "loss": 1.806, "theoretical_loss": 4.621255337563955, "tokens_seen": 132579328 }, { "epoch": 0.04, "learning_rate": 0.0007756068204613841, "loss": 1.8109, "theoretical_loss": 4.620953164144376, "tokens_seen": 132644864 }, { "epoch": 0.04, "learning_rate": 0.0007755907723169509, "loss": 1.8321, "theoretical_loss": 4.620651181762212, "tokens_seen": 132710400 }, { "epoch": 0.04, "learning_rate": 0.0007755747241725175, "loss": 1.8282, "theoretical_loss": 4.620349390202436, "tokens_seen": 132775936 }, { "epoch": 0.04, "learning_rate": 0.0007755586760280843, "loss": 1.8138, "theoretical_loss": 4.6200477892503695, "tokens_seen": 132841472 }, { "epoch": 0.04, "learning_rate": 0.0007755426278836509, "loss": 1.8533, "theoretical_loss": 4.619746378691682, "tokens_seen": 132907008 }, { "epoch": 0.04, "learning_rate": 0.0007755265797392177, "loss": 1.7597, "theoretical_loss": 4.61944515831239, "tokens_seen": 132972544 }, { "epoch": 0.04, "learning_rate": 0.0007755105315947843, "loss": 1.8646, "theoretical_loss": 4.6191441278988545, "tokens_seen": 133038080 }, { "epoch": 0.04, "learning_rate": 0.0007754944834503511, "loss": 1.7799, "theoretical_loss": 4.618843287237782, "tokens_seen": 133103616 }, { "epoch": 0.04, "learning_rate": 0.0007754784353059177, "loss": 1.8689, "theoretical_loss": 4.618542636116224, "tokens_seen": 133169152 }, { "epoch": 0.04, "learning_rate": 0.0007754623871614845, "loss": 1.833, "theoretical_loss": 4.618242174321575, "tokens_seen": 133234688 }, { "epoch": 0.04, "learning_rate": 0.0007754463390170511, "loss": 1.8088, "theoretical_loss": 4.617941901641573, "tokens_seen": 133300224 }, { "epoch": 0.04, "learning_rate": 0.0007754302908726179, "loss": 1.8624, "theoretical_loss": 4.617641817864296, "tokens_seen": 133365760 }, { "epoch": 0.04, "learning_rate": 0.0007754142427281845, "loss": 1.9235, "theoretical_loss": 4.617341922778168, "tokens_seen": 133431296 }, { "epoch": 0.04, "learning_rate": 0.0007753981945837513, "loss": 1.701, "theoretical_loss": 4.617042216171949, "tokens_seen": 133496832 }, { "epoch": 0.04, "learning_rate": 0.0007753821464393179, "loss": 1.7201, "theoretical_loss": 4.616742697834742, "tokens_seen": 133562368 }, { "epoch": 0.04, "learning_rate": 0.0007753660982948847, "loss": 1.8286, "theoretical_loss": 4.616443367555988, "tokens_seen": 133627904 }, { "epoch": 0.04, "learning_rate": 0.0007753500501504513, "loss": 1.8104, "theoretical_loss": 4.616144225125465, "tokens_seen": 133693440 }, { "epoch": 0.04, "learning_rate": 0.0007753340020060181, "loss": 1.8212, "theoretical_loss": 4.615845270333294, "tokens_seen": 133758976 }, { "epoch": 0.04, "learning_rate": 0.0007753179538615847, "loss": 1.7838, "theoretical_loss": 4.615546502969929, "tokens_seen": 133824512 }, { "epoch": 0.04, "learning_rate": 0.0007753019057171515, "loss": 1.8179, "theoretical_loss": 4.61524792282616, "tokens_seen": 133890048 }, { "epoch": 0.04, "learning_rate": 0.0007752858575727181, "loss": 1.8488, "theoretical_loss": 4.614949529693115, "tokens_seen": 133955584 }, { "epoch": 0.04, "learning_rate": 0.0007752698094282849, "loss": 1.7209, "theoretical_loss": 4.614651323362257, "tokens_seen": 134021120 }, { "epoch": 0.04, "learning_rate": 0.0007752537612838515, "loss": 1.6942, "theoretical_loss": 4.614353303625382, "tokens_seen": 134086656 }, { "epoch": 0.04, "learning_rate": 0.0007752377131394183, "loss": 1.819, "theoretical_loss": 4.6140554702746215, "tokens_seen": 134152192 }, { "epoch": 0.04, "learning_rate": 0.000775221664994985, "loss": 1.7107, "theoretical_loss": 4.613757823102437, "tokens_seen": 134217728 }, { "epoch": 0.04, "learning_rate": 0.0007752056168505517, "loss": 1.782, "theoretical_loss": 4.613460361901627, "tokens_seen": 134283264 }, { "epoch": 0.04, "objective/train/docs_used": 51744, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.0655648708343506, "objective/train/theoretical_loss": 4.613163086465315, "objective/train/tokens_used": 154808800, "theoretical_loss": 4.613163086465315, "tokens_seen": 134348800 }, { "epoch": 0.04, "learning_rate": 0.0007751895687061184, "loss": 1.9222, "theoretical_loss": 4.613163086465315, "tokens_seen": 134348800 }, { "epoch": 0.04, "learning_rate": 0.0007751735205616851, "loss": 1.7955, "theoretical_loss": 4.612865996586962, "tokens_seen": 134414336 }, { "epoch": 0.04, "learning_rate": 0.0007751574724172518, "loss": 1.7857, "theoretical_loss": 4.612569092060355, "tokens_seen": 134479872 }, { "epoch": 0.04, "learning_rate": 0.0007751414242728185, "loss": 1.8141, "theoretical_loss": 4.612272372679611, "tokens_seen": 134545408 }, { "epoch": 0.04, "learning_rate": 0.0007751253761283852, "loss": 1.9289, "theoretical_loss": 4.611975838239179, "tokens_seen": 134610944 }, { "epoch": 0.04, "learning_rate": 0.0007751093279839519, "loss": 1.6775, "theoretical_loss": 4.611679488533832, "tokens_seen": 134676480 }, { "epoch": 0.04, "learning_rate": 0.0007750932798395186, "loss": 1.8499, "theoretical_loss": 4.611383323358671, "tokens_seen": 134742016 }, { "epoch": 0.04, "learning_rate": 0.0007750772316950853, "loss": 1.8098, "theoretical_loss": 4.61108734250913, "tokens_seen": 134807552 }, { "epoch": 0.04, "learning_rate": 0.000775061183550652, "loss": 1.8215, "theoretical_loss": 4.610791545780959, "tokens_seen": 134873088 }, { "epoch": 0.04, "learning_rate": 0.0007750451354062187, "loss": 1.6756, "theoretical_loss": 4.610495932970241, "tokens_seen": 134938624 }, { "epoch": 0.04, "learning_rate": 0.0007750290872617854, "loss": 1.7515, "theoretical_loss": 4.61020050387338, "tokens_seen": 135004160 }, { "epoch": 0.04, "learning_rate": 0.0007750130391173521, "loss": 1.7896, "theoretical_loss": 4.609905258287107, "tokens_seen": 135069696 }, { "epoch": 0.04, "learning_rate": 0.0007749969909729188, "loss": 1.8716, "theoretical_loss": 4.609610196008473, "tokens_seen": 135135232 }, { "epoch": 0.04, "learning_rate": 0.0007749809428284855, "loss": 1.7909, "theoretical_loss": 4.609315316834855, "tokens_seen": 135200768 }, { "epoch": 0.04, "learning_rate": 0.0007749648946840522, "loss": 1.7184, "theoretical_loss": 4.60902062056395, "tokens_seen": 135266304 }, { "epoch": 0.04, "learning_rate": 0.0007749488465396189, "loss": 1.8287, "theoretical_loss": 4.608726106993776, "tokens_seen": 135331840 }, { "epoch": 0.04, "learning_rate": 0.0007749327983951856, "loss": 1.9142, "theoretical_loss": 4.608431775922673, "tokens_seen": 135397376 }, { "epoch": 0.04, "learning_rate": 0.0007749167502507523, "loss": 1.7733, "theoretical_loss": 4.608137627149302, "tokens_seen": 135462912 }, { "epoch": 0.04, "learning_rate": 0.000774900702106319, "loss": 1.8085, "theoretical_loss": 4.607843660472642, "tokens_seen": 135528448 }, { "epoch": 0.04, "learning_rate": 0.0007748846539618857, "loss": 1.8051, "theoretical_loss": 4.607549875691989, "tokens_seen": 135593984 }, { "epoch": 0.04, "learning_rate": 0.0007748686058174524, "loss": 1.8313, "theoretical_loss": 4.607256272606961, "tokens_seen": 135659520 }, { "epoch": 0.04, "learning_rate": 0.0007748525576730191, "loss": 1.8189, "theoretical_loss": 4.606962851017489, "tokens_seen": 135725056 }, { "epoch": 0.04, "learning_rate": 0.0007748365095285858, "loss": 1.905, "theoretical_loss": 4.606669610723825, "tokens_seen": 135790592 }, { "epoch": 0.04, "learning_rate": 0.0007748204613841525, "loss": 1.8924, "theoretical_loss": 4.606376551526536, "tokens_seen": 135856128 }, { "epoch": 0.04, "learning_rate": 0.0007748044132397192, "loss": 1.7682, "theoretical_loss": 4.6060836732265, "tokens_seen": 135921664 }, { "epoch": 0.04, "learning_rate": 0.0007747883650952859, "loss": 1.7632, "theoretical_loss": 4.605790975624917, "tokens_seen": 135987200 }, { "epoch": 0.04, "learning_rate": 0.0007747723169508527, "loss": 1.8681, "theoretical_loss": 4.605498458523296, "tokens_seen": 136052736 }, { "epoch": 0.04, "learning_rate": 0.0007747562688064193, "loss": 1.7845, "theoretical_loss": 4.605206121723461, "tokens_seen": 136118272 }, { "epoch": 0.04, "learning_rate": 0.0007747402206619861, "loss": 1.7568, "theoretical_loss": 4.604913965027551, "tokens_seen": 136183808 }, { "epoch": 0.04, "learning_rate": 0.0007747241725175527, "loss": 1.8752, "theoretical_loss": 4.604621988238012, "tokens_seen": 136249344 }, { "epoch": 0.04, "learning_rate": 0.0007747081243731195, "loss": 1.9572, "theoretical_loss": 4.604330191157608, "tokens_seen": 136314880 }, { "epoch": 0.04, "learning_rate": 0.0007746920762286861, "loss": 1.7565, "theoretical_loss": 4.604038573589408, "tokens_seen": 136380416 }, { "epoch": 0.04, "learning_rate": 0.0007746760280842529, "loss": 1.783, "theoretical_loss": 4.603747135336796, "tokens_seen": 136445952 }, { "epoch": 0.04, "learning_rate": 0.0007746599799398195, "loss": 1.8388, "theoretical_loss": 4.603455876203463, "tokens_seen": 136511488 }, { "epoch": 0.04, "learning_rate": 0.0007746439317953863, "loss": 1.7053, "theoretical_loss": 4.603164795993409, "tokens_seen": 136577024 }, { "epoch": 0.04, "learning_rate": 0.0007746278836509529, "loss": 1.7839, "theoretical_loss": 4.602873894510945, "tokens_seen": 136642560 }, { "epoch": 0.04, "learning_rate": 0.0007746118355065197, "loss": 1.7043, "theoretical_loss": 4.602583171560686, "tokens_seen": 136708096 }, { "epoch": 0.04, "learning_rate": 0.0007745957873620863, "loss": 1.7794, "theoretical_loss": 4.602292626947556, "tokens_seen": 136773632 }, { "epoch": 0.04, "learning_rate": 0.0007745797392176531, "loss": 1.8616, "theoretical_loss": 4.602002260476787, "tokens_seen": 136839168 }, { "epoch": 0.04, "learning_rate": 0.0007745636910732197, "loss": 1.783, "theoretical_loss": 4.601712071953914, "tokens_seen": 136904704 }, { "epoch": 0.04, "learning_rate": 0.0007745476429287865, "loss": 1.7862, "theoretical_loss": 4.601422061184778, "tokens_seen": 136970240 }, { "epoch": 0.04, "learning_rate": 0.0007745315947843531, "loss": 1.6995, "theoretical_loss": 4.601132227975527, "tokens_seen": 137035776 }, { "epoch": 0.04, "learning_rate": 0.0007745155466399198, "loss": 2.0007, "theoretical_loss": 4.60084257213261, "tokens_seen": 137101312 }, { "epoch": 0.04, "learning_rate": 0.0007744994984954865, "loss": 1.7742, "theoretical_loss": 4.600553093462781, "tokens_seen": 137166848 }, { "epoch": 0.04, "learning_rate": 0.0007744834503510532, "loss": 1.9237, "theoretical_loss": 4.600263791773097, "tokens_seen": 137232384 }, { "epoch": 0.04, "learning_rate": 0.0007744674022066199, "loss": 1.8399, "theoretical_loss": 4.599974666870914, "tokens_seen": 137297920 }, { "epoch": 0.04, "learning_rate": 0.0007744513540621866, "loss": 1.7832, "theoretical_loss": 4.599685718563894, "tokens_seen": 137363456 }, { "epoch": 0.04, "learning_rate": 0.0007744353059177533, "loss": 1.801, "theoretical_loss": 4.599396946659997, "tokens_seen": 137428992 }, { "epoch": 0.04, "learning_rate": 0.00077441925777332, "loss": 1.7621, "theoretical_loss": 4.5991083509674855, "tokens_seen": 137494528 }, { "epoch": 0.04, "learning_rate": 0.0007744032096288867, "loss": 1.7486, "theoretical_loss": 4.598819931294919, "tokens_seen": 137560064 }, { "epoch": 0.04, "objective/train/docs_used": 52717, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7819393873214722, "objective/train/theoretical_loss": 4.598531687451157, "objective/train/tokens_used": 158085600, "theoretical_loss": 4.598531687451157, "tokens_seen": 137625600 }, { "epoch": 0.04, "learning_rate": 0.0007743871614844534, "loss": 1.8069, "theoretical_loss": 4.598531687451157, "tokens_seen": 137625600 }, { "epoch": 0.04, "learning_rate": 0.00077437111334002, "loss": 1.8632, "theoretical_loss": 4.598243619245359, "tokens_seen": 137691136 }, { "epoch": 0.04, "learning_rate": 0.0007743550651955868, "loss": 1.8544, "theoretical_loss": 4.59795572648698, "tokens_seen": 137756672 }, { "epoch": 0.04, "learning_rate": 0.0007743390170511534, "loss": 1.8115, "theoretical_loss": 4.597668008985774, "tokens_seen": 137822208 }, { "epoch": 0.04, "learning_rate": 0.0007743229689067202, "loss": 1.7225, "theoretical_loss": 4.59738046655179, "tokens_seen": 137887744 }, { "epoch": 0.04, "learning_rate": 0.0007743069207622868, "loss": 1.7712, "theoretical_loss": 4.597093098995377, "tokens_seen": 137953280 }, { "epoch": 0.04, "learning_rate": 0.0007742908726178536, "loss": 1.735, "theoretical_loss": 4.596805906127173, "tokens_seen": 138018816 }, { "epoch": 0.04, "learning_rate": 0.0007742748244734202, "loss": 1.7213, "theoretical_loss": 4.596518887758117, "tokens_seen": 138084352 }, { "epoch": 0.04, "learning_rate": 0.000774258776328987, "loss": 1.867, "theoretical_loss": 4.596232043699438, "tokens_seen": 138149888 }, { "epoch": 0.04, "learning_rate": 0.0007742427281845536, "loss": 1.842, "theoretical_loss": 4.59594537376266, "tokens_seen": 138215424 }, { "epoch": 0.04, "learning_rate": 0.0007742266800401204, "loss": 1.8787, "theoretical_loss": 4.595658877759602, "tokens_seen": 138280960 }, { "epoch": 0.04, "learning_rate": 0.000774210631895687, "loss": 1.9223, "theoretical_loss": 4.595372555502371, "tokens_seen": 138346496 }, { "epoch": 0.04, "learning_rate": 0.0007741945837512538, "loss": 1.9055, "theoretical_loss": 4.59508640680337, "tokens_seen": 138412032 }, { "epoch": 0.04, "learning_rate": 0.0007741785356068204, "loss": 1.8217, "theoretical_loss": 4.594800431475292, "tokens_seen": 138477568 }, { "epoch": 0.04, "learning_rate": 0.0007741624874623872, "loss": 1.7421, "theoretical_loss": 4.594514629331121, "tokens_seen": 138543104 }, { "epoch": 0.04, "learning_rate": 0.0007741464393179538, "loss": 1.7933, "theoretical_loss": 4.594229000184128, "tokens_seen": 138608640 }, { "epoch": 0.04, "learning_rate": 0.0007741303911735206, "loss": 1.8861, "theoretical_loss": 4.593943543847878, "tokens_seen": 138674176 }, { "epoch": 0.04, "learning_rate": 0.0007741143430290872, "loss": 1.8354, "theoretical_loss": 4.5936582601362215, "tokens_seen": 138739712 }, { "epoch": 0.04, "learning_rate": 0.000774098294884654, "loss": 1.8875, "theoretical_loss": 4.593373148863301, "tokens_seen": 138805248 }, { "epoch": 0.04, "learning_rate": 0.0007740822467402206, "loss": 1.8701, "theoretical_loss": 4.593088209843543, "tokens_seen": 138870784 }, { "epoch": 0.04, "learning_rate": 0.0007740661985957874, "loss": 1.8651, "theoretical_loss": 4.592803442891663, "tokens_seen": 138936320 }, { "epoch": 0.04, "learning_rate": 0.000774050150451354, "loss": 1.8731, "theoretical_loss": 4.592518847822661, "tokens_seen": 139001856 }, { "epoch": 0.04, "learning_rate": 0.0007740341023069208, "loss": 1.7787, "theoretical_loss": 4.592234424451828, "tokens_seen": 139067392 }, { "epoch": 0.04, "learning_rate": 0.0007740180541624874, "loss": 1.8654, "theoretical_loss": 4.591950172594734, "tokens_seen": 139132928 }, { "epoch": 0.04, "learning_rate": 0.0007740020060180542, "loss": 1.728, "theoretical_loss": 4.591666092067238, "tokens_seen": 139198464 }, { "epoch": 0.04, "learning_rate": 0.0007739859578736208, "loss": 1.8389, "theoretical_loss": 4.591382182685484, "tokens_seen": 139264000 }, { "epoch": 0.04, "learning_rate": 0.0007739699097291876, "loss": 1.7957, "theoretical_loss": 4.591098444265896, "tokens_seen": 139329536 }, { "epoch": 0.04, "learning_rate": 0.0007739538615847542, "loss": 1.7924, "theoretical_loss": 4.590814876625183, "tokens_seen": 139395072 }, { "epoch": 0.04, "learning_rate": 0.000773937813440321, "loss": 1.8632, "theoretical_loss": 4.590531479580338, "tokens_seen": 139460608 }, { "epoch": 0.04, "learning_rate": 0.0007739217652958876, "loss": 1.747, "theoretical_loss": 4.5902482529486335, "tokens_seen": 139526144 }, { "epoch": 0.04, "learning_rate": 0.0007739057171514544, "loss": 1.8658, "theoretical_loss": 4.589965196547625, "tokens_seen": 139591680 }, { "epoch": 0.04, "learning_rate": 0.000773889669007021, "loss": 1.904, "theoretical_loss": 4.5896823101951485, "tokens_seen": 139657216 }, { "epoch": 0.04, "learning_rate": 0.0007738736208625878, "loss": 1.8037, "theoretical_loss": 4.58939959370932, "tokens_seen": 139722752 }, { "epoch": 0.04, "learning_rate": 0.0007738575727181545, "loss": 1.7745, "theoretical_loss": 4.5891170469085365, "tokens_seen": 139788288 }, { "epoch": 0.04, "learning_rate": 0.0007738415245737212, "loss": 1.7752, "theoretical_loss": 4.588834669611472, "tokens_seen": 139853824 }, { "epoch": 0.04, "learning_rate": 0.0007738254764292879, "loss": 1.8438, "theoretical_loss": 4.5885524616370805, "tokens_seen": 139919360 }, { "epoch": 0.04, "learning_rate": 0.0007738094282848546, "loss": 1.8438, "theoretical_loss": 4.5882704228045945, "tokens_seen": 139984896 }, { "epoch": 0.04, "learning_rate": 0.0007737933801404213, "loss": 1.916, "theoretical_loss": 4.587988552933523, "tokens_seen": 140050432 }, { "epoch": 0.04, "learning_rate": 0.000773777331995988, "loss": 1.7058, "theoretical_loss": 4.587706851843652, "tokens_seen": 140115968 }, { "epoch": 0.04, "learning_rate": 0.0007737612838515547, "loss": 1.8099, "theoretical_loss": 4.587425319355044, "tokens_seen": 140181504 }, { "epoch": 0.04, "learning_rate": 0.0007737452357071214, "loss": 1.7592, "theoretical_loss": 4.587143955288038, "tokens_seen": 140247040 }, { "epoch": 0.04, "learning_rate": 0.0007737291875626881, "loss": 1.8013, "theoretical_loss": 4.586862759463248, "tokens_seen": 140312576 }, { "epoch": 0.04, "learning_rate": 0.0007737131394182548, "loss": 1.8653, "theoretical_loss": 4.586581731701562, "tokens_seen": 140378112 }, { "epoch": 0.04, "learning_rate": 0.0007736970912738215, "loss": 1.8956, "theoretical_loss": 4.586300871824143, "tokens_seen": 140443648 }, { "epoch": 0.04, "learning_rate": 0.0007736810431293882, "loss": 1.8632, "theoretical_loss": 4.586020179652427, "tokens_seen": 140509184 }, { "epoch": 0.04, "learning_rate": 0.0007736649949849549, "loss": 1.7402, "theoretical_loss": 4.585739655008123, "tokens_seen": 140574720 }, { "epoch": 0.04, "learning_rate": 0.0007736489468405216, "loss": 1.8236, "theoretical_loss": 4.585459297713215, "tokens_seen": 140640256 }, { "epoch": 0.04, "learning_rate": 0.0007736328986960883, "loss": 1.8042, "theoretical_loss": 4.585179107589955, "tokens_seen": 140705792 }, { "epoch": 0.04, "learning_rate": 0.000773616850551655, "loss": 1.8166, "theoretical_loss": 4.58489908446087, "tokens_seen": 140771328 }, { "epoch": 0.04, "learning_rate": 0.0007736008024072217, "loss": 1.7575, "theoretical_loss": 4.584619228148755, "tokens_seen": 140836864 }, { "epoch": 0.04, "objective/train/docs_used": 53917, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8665790557861328, "objective/train/theoretical_loss": 4.584339538476678, "objective/train/tokens_used": 161362400, "theoretical_loss": 4.584339538476678, "tokens_seen": 140902400 }, { "epoch": 0.04, "learning_rate": 0.0007735847542627884, "loss": 1.8941, "theoretical_loss": 4.584339538476678, "tokens_seen": 140902400 }, { "epoch": 0.04, "learning_rate": 0.0007735687061183552, "loss": 1.6903, "theoretical_loss": 4.5840600152679745, "tokens_seen": 140967936 }, { "epoch": 0.04, "learning_rate": 0.0007735526579739218, "loss": 1.8307, "theoretical_loss": 4.583780658346252, "tokens_seen": 141033472 }, { "epoch": 0.04, "learning_rate": 0.0007735366098294886, "loss": 1.7627, "theoretical_loss": 4.583501467535383, "tokens_seen": 141099008 }, { "epoch": 0.04, "learning_rate": 0.0007735205616850552, "loss": 1.6755, "theoretical_loss": 4.583222442659514, "tokens_seen": 141164544 }, { "epoch": 0.04, "learning_rate": 0.000773504513540622, "loss": 1.8388, "theoretical_loss": 4.5829435835430505, "tokens_seen": 141230080 }, { "epoch": 0.04, "learning_rate": 0.0007734884653961886, "loss": 1.833, "theoretical_loss": 4.582664890010674, "tokens_seen": 141295616 }, { "epoch": 0.04, "learning_rate": 0.0007734724172517554, "loss": 1.9159, "theoretical_loss": 4.582386361887329, "tokens_seen": 141361152 }, { "epoch": 0.04, "learning_rate": 0.000773456369107322, "loss": 1.7725, "theoretical_loss": 4.582107998998225, "tokens_seen": 141426688 }, { "epoch": 0.04, "learning_rate": 0.0007734403209628888, "loss": 1.8281, "theoretical_loss": 4.581829801168839, "tokens_seen": 141492224 }, { "epoch": 0.04, "learning_rate": 0.0007734242728184554, "loss": 1.7452, "theoretical_loss": 4.5815517682249105, "tokens_seen": 141557760 }, { "epoch": 0.04, "learning_rate": 0.0007734082246740222, "loss": 1.8201, "theoretical_loss": 4.581273899992447, "tokens_seen": 141623296 }, { "epoch": 0.04, "learning_rate": 0.0007733921765295888, "loss": 1.8788, "theoretical_loss": 4.580996196297718, "tokens_seen": 141688832 }, { "epoch": 0.04, "learning_rate": 0.0007733761283851556, "loss": 1.8429, "theoretical_loss": 4.580718656967257, "tokens_seen": 141754368 }, { "epoch": 0.04, "learning_rate": 0.0007733600802407222, "loss": 1.8612, "theoretical_loss": 4.580441281827858, "tokens_seen": 141819904 }, { "epoch": 0.04, "learning_rate": 0.000773344032096289, "loss": 1.7208, "theoretical_loss": 4.580164070706584, "tokens_seen": 141885440 }, { "epoch": 0.04, "learning_rate": 0.0007733279839518556, "loss": 1.7845, "theoretical_loss": 4.579887023430752, "tokens_seen": 141950976 }, { "epoch": 0.04, "learning_rate": 0.0007733119358074224, "loss": 1.7119, "theoretical_loss": 4.579610139827945, "tokens_seen": 142016512 }, { "epoch": 0.04, "learning_rate": 0.000773295887662989, "loss": 1.7464, "theoretical_loss": 4.579333419726007, "tokens_seen": 142082048 }, { "epoch": 0.04, "learning_rate": 0.0007732798395185558, "loss": 1.8061, "theoretical_loss": 4.579056862953039, "tokens_seen": 142147584 }, { "epoch": 0.04, "learning_rate": 0.0007732637913741224, "loss": 1.8128, "theoretical_loss": 4.578780469337407, "tokens_seen": 142213120 }, { "epoch": 0.04, "learning_rate": 0.0007732477432296892, "loss": 1.77, "theoretical_loss": 4.5785042387077315, "tokens_seen": 142278656 }, { "epoch": 0.04, "learning_rate": 0.0007732316950852558, "loss": 1.8202, "theoretical_loss": 4.578228170892896, "tokens_seen": 142344192 }, { "epoch": 0.04, "learning_rate": 0.0007732156469408226, "loss": 1.9073, "theoretical_loss": 4.577952265722038, "tokens_seen": 142409728 }, { "epoch": 0.04, "learning_rate": 0.0007731995987963892, "loss": 1.8577, "theoretical_loss": 4.5776765230245555, "tokens_seen": 142475264 }, { "epoch": 0.04, "learning_rate": 0.000773183550651956, "loss": 1.8928, "theoretical_loss": 4.5774009426301046, "tokens_seen": 142540800 }, { "epoch": 0.04, "learning_rate": 0.0007731675025075226, "loss": 1.8294, "theoretical_loss": 4.577125524368595, "tokens_seen": 142606336 }, { "epoch": 0.04, "learning_rate": 0.0007731514543630894, "loss": 1.7549, "theoretical_loss": 4.5768502680701975, "tokens_seen": 142671872 }, { "epoch": 0.04, "learning_rate": 0.000773135406218656, "loss": 1.8273, "theoretical_loss": 4.576575173565334, "tokens_seen": 142737408 }, { "epoch": 0.04, "learning_rate": 0.0007731193580742227, "loss": 1.8096, "theoretical_loss": 4.5763002406846836, "tokens_seen": 142802944 }, { "epoch": 0.04, "learning_rate": 0.0007731033099297894, "loss": 1.8756, "theoretical_loss": 4.57602546925918, "tokens_seen": 142868480 }, { "epoch": 0.04, "learning_rate": 0.0007730872617853561, "loss": 1.6733, "theoretical_loss": 4.575750859120014, "tokens_seen": 142934016 }, { "epoch": 0.04, "learning_rate": 0.0007730712136409228, "loss": 1.8921, "theoretical_loss": 4.575476410098624, "tokens_seen": 142999552 }, { "epoch": 0.04, "learning_rate": 0.0007730551654964895, "loss": 1.7704, "theoretical_loss": 4.575202122026706, "tokens_seen": 143065088 }, { "epoch": 0.04, "learning_rate": 0.0007730391173520562, "loss": 1.773, "theoretical_loss": 4.5749279947362105, "tokens_seen": 143130624 }, { "epoch": 0.04, "learning_rate": 0.0007730230692076229, "loss": 1.8325, "theoretical_loss": 4.574654028059336, "tokens_seen": 143196160 }, { "epoch": 0.04, "learning_rate": 0.0007730070210631896, "loss": 1.8279, "theoretical_loss": 4.574380221828535, "tokens_seen": 143261696 }, { "epoch": 0.04, "learning_rate": 0.0007729909729187563, "loss": 1.7535, "theoretical_loss": 4.574106575876511, "tokens_seen": 143327232 }, { "epoch": 0.04, "learning_rate": 0.0007729749247743229, "loss": 1.8514, "theoretical_loss": 4.573833090036219, "tokens_seen": 143392768 }, { "epoch": 0.04, "learning_rate": 0.0007729588766298897, "loss": 1.8637, "theoretical_loss": 4.573559764140864, "tokens_seen": 143458304 }, { "epoch": 0.04, "learning_rate": 0.0007729428284854563, "loss": 1.6962, "theoretical_loss": 4.573286598023898, "tokens_seen": 143523840 }, { "epoch": 0.04, "learning_rate": 0.0007729267803410231, "loss": 1.8597, "theoretical_loss": 4.573013591519029, "tokens_seen": 143589376 }, { "epoch": 0.04, "learning_rate": 0.0007729107321965897, "loss": 1.9061, "theoretical_loss": 4.572740744460207, "tokens_seen": 143654912 }, { "epoch": 0.04, "learning_rate": 0.0007728946840521565, "loss": 1.9639, "theoretical_loss": 4.572468056681634, "tokens_seen": 143720448 }, { "epoch": 0.04, "learning_rate": 0.0007728786359077231, "loss": 1.8062, "theoretical_loss": 4.57219552801776, "tokens_seen": 143785984 }, { "epoch": 0.04, "learning_rate": 0.0007728625877632899, "loss": 1.791, "theoretical_loss": 4.57192315830328, "tokens_seen": 143851520 }, { "epoch": 0.04, "learning_rate": 0.0007728465396188565, "loss": 1.7563, "theoretical_loss": 4.571650947373138, "tokens_seen": 143917056 }, { "epoch": 0.04, "learning_rate": 0.0007728304914744233, "loss": 1.8727, "theoretical_loss": 4.571378895062525, "tokens_seen": 143982592 }, { "epoch": 0.04, "learning_rate": 0.0007728144433299899, "loss": 1.7582, "theoretical_loss": 4.571107001206875, "tokens_seen": 144048128 }, { "epoch": 0.04, "learning_rate": 0.0007727983951855567, "loss": 1.885, "theoretical_loss": 4.570835265641872, "tokens_seen": 144113664 }, { "epoch": 0.04, "objective/train/docs_used": 54793, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8981997966766357, "objective/train/theoretical_loss": 4.57056368820344, "objective/train/tokens_used": 164639200, "theoretical_loss": 4.57056368820344, "tokens_seen": 144179200 }, { "epoch": 0.04, "learning_rate": 0.0007727823470411233, "loss": 1.7713, "theoretical_loss": 4.57056368820344, "tokens_seen": 144179200 }, { "epoch": 0.04, "learning_rate": 0.0007727662988966901, "loss": 1.722, "theoretical_loss": 4.570292268727751, "tokens_seen": 144244736 }, { "epoch": 0.04, "learning_rate": 0.0007727502507522567, "loss": 1.7943, "theoretical_loss": 4.570021007051221, "tokens_seen": 144310272 }, { "epoch": 0.04, "learning_rate": 0.0007727342026078235, "loss": 1.7481, "theoretical_loss": 4.569749903010507, "tokens_seen": 144375808 }, { "epoch": 0.04, "learning_rate": 0.0007727181544633901, "loss": 1.7384, "theoretical_loss": 4.569478956442513, "tokens_seen": 144441344 }, { "epoch": 0.04, "learning_rate": 0.0007727021063189569, "loss": 2.0361, "theoretical_loss": 4.569208167184382, "tokens_seen": 144506880 }, { "epoch": 0.04, "learning_rate": 0.0007726860581745235, "loss": 1.7303, "theoretical_loss": 4.5689375350735, "tokens_seen": 144572416 }, { "epoch": 0.04, "learning_rate": 0.0007726700100300903, "loss": 1.7676, "theoretical_loss": 4.568667059947496, "tokens_seen": 144637952 }, { "epoch": 0.04, "learning_rate": 0.0007726539618856569, "loss": 1.8165, "theoretical_loss": 4.568396741644241, "tokens_seen": 144703488 }, { "epoch": 0.04, "learning_rate": 0.0007726379137412237, "loss": 1.8475, "theoretical_loss": 4.568126580001843, "tokens_seen": 144769024 }, { "epoch": 0.04, "learning_rate": 0.0007726218655967903, "loss": 1.6271, "theoretical_loss": 4.567856574858656, "tokens_seen": 144834560 }, { "epoch": 0.04, "learning_rate": 0.0007726058174523571, "loss": 1.9387, "theoretical_loss": 4.567586726053266, "tokens_seen": 144900096 }, { "epoch": 0.04, "learning_rate": 0.0007725897693079237, "loss": 1.7635, "theoretical_loss": 4.567317033424507, "tokens_seen": 144965632 }, { "epoch": 0.04, "learning_rate": 0.0007725737211634905, "loss": 1.7297, "theoretical_loss": 4.567047496811445, "tokens_seen": 145031168 }, { "epoch": 0.04, "learning_rate": 0.0007725576730190571, "loss": 1.8948, "theoretical_loss": 4.566778116053389, "tokens_seen": 145096704 }, { "epoch": 0.04, "learning_rate": 0.0007725416248746239, "loss": 1.9421, "theoretical_loss": 4.566508890989883, "tokens_seen": 145162240 }, { "epoch": 0.04, "learning_rate": 0.0007725255767301905, "loss": 1.8752, "theoretical_loss": 4.566239821460712, "tokens_seen": 145227776 }, { "epoch": 0.04, "learning_rate": 0.0007725095285857573, "loss": 1.7984, "theoretical_loss": 4.5659709073058945, "tokens_seen": 145293312 }, { "epoch": 0.04, "learning_rate": 0.000772493480441324, "loss": 1.8638, "theoretical_loss": 4.565702148365686, "tokens_seen": 145358848 }, { "epoch": 0.04, "learning_rate": 0.0007724774322968907, "loss": 1.7664, "theoretical_loss": 4.56543354448058, "tokens_seen": 145424384 }, { "epoch": 0.04, "learning_rate": 0.0007724613841524574, "loss": 1.842, "theoretical_loss": 4.565165095491306, "tokens_seen": 145489920 }, { "epoch": 0.04, "learning_rate": 0.0007724453360080241, "loss": 1.7626, "theoretical_loss": 4.5648968012388265, "tokens_seen": 145555456 }, { "epoch": 0.04, "learning_rate": 0.0007724292878635908, "loss": 1.7205, "theoretical_loss": 4.56462866156434, "tokens_seen": 145620992 }, { "epoch": 0.04, "learning_rate": 0.0007724132397191575, "loss": 1.7857, "theoretical_loss": 4.564360676309279, "tokens_seen": 145686528 }, { "epoch": 0.04, "learning_rate": 0.0007723971915747242, "loss": 1.8482, "theoretical_loss": 4.564092845315312, "tokens_seen": 145752064 }, { "epoch": 0.04, "learning_rate": 0.0007723811434302909, "loss": 1.7422, "theoretical_loss": 4.563825168424337, "tokens_seen": 145817600 }, { "epoch": 0.04, "learning_rate": 0.0007723650952858577, "loss": 1.7175, "theoretical_loss": 4.563557645478488, "tokens_seen": 145883136 }, { "epoch": 0.04, "learning_rate": 0.0007723490471414243, "loss": 1.8653, "theoretical_loss": 4.56329027632013, "tokens_seen": 145948672 }, { "epoch": 0.04, "learning_rate": 0.0007723329989969911, "loss": 1.7809, "theoretical_loss": 4.563023060791862, "tokens_seen": 146014208 }, { "epoch": 0.04, "learning_rate": 0.0007723169508525577, "loss": 1.729, "theoretical_loss": 4.562755998736511, "tokens_seen": 146079744 }, { "epoch": 0.04, "learning_rate": 0.0007723009027081245, "loss": 1.7853, "theoretical_loss": 4.562489089997141, "tokens_seen": 146145280 }, { "epoch": 0.04, "learning_rate": 0.0007722848545636911, "loss": 1.7846, "theoretical_loss": 4.562222334417041, "tokens_seen": 146210816 }, { "epoch": 0.04, "learning_rate": 0.0007722688064192579, "loss": 1.7191, "theoretical_loss": 4.561955731839734, "tokens_seen": 146276352 }, { "epoch": 0.04, "learning_rate": 0.0007722527582748245, "loss": 1.8433, "theoretical_loss": 4.561689282108971, "tokens_seen": 146341888 }, { "epoch": 0.04, "learning_rate": 0.0007722367101303913, "loss": 1.818, "theoretical_loss": 4.561422985068733, "tokens_seen": 146407424 }, { "epoch": 0.04, "learning_rate": 0.0007722206619859579, "loss": 1.7675, "theoretical_loss": 4.561156840563231, "tokens_seen": 146472960 }, { "epoch": 0.04, "learning_rate": 0.0007722046138415247, "loss": 1.836, "theoretical_loss": 4.560890848436902, "tokens_seen": 146538496 }, { "epoch": 0.04, "learning_rate": 0.0007721885656970913, "loss": 1.8614, "theoretical_loss": 4.560625008534414, "tokens_seen": 146604032 }, { "epoch": 0.04, "learning_rate": 0.0007721725175526581, "loss": 1.824, "theoretical_loss": 4.560359320700662, "tokens_seen": 146669568 }, { "epoch": 0.04, "learning_rate": 0.0007721564694082247, "loss": 1.7436, "theoretical_loss": 4.560093784780766, "tokens_seen": 146735104 }, { "epoch": 0.04, "learning_rate": 0.0007721404212637915, "loss": 1.8323, "theoretical_loss": 4.559828400620075, "tokens_seen": 146800640 }, { "epoch": 0.04, "learning_rate": 0.0007721243731193581, "loss": 1.89, "theoretical_loss": 4.559563168064165, "tokens_seen": 146866176 }, { "epoch": 0.04, "learning_rate": 0.0007721083249749249, "loss": 1.9487, "theoretical_loss": 4.559298086958837, "tokens_seen": 146931712 }, { "epoch": 0.04, "learning_rate": 0.0007720922768304915, "loss": 1.6885, "theoretical_loss": 4.559033157150115, "tokens_seen": 146997248 }, { "epoch": 0.04, "learning_rate": 0.0007720762286860583, "loss": 1.8212, "theoretical_loss": 4.5587683784842525, "tokens_seen": 147062784 }, { "epoch": 0.04, "learning_rate": 0.0007720601805416249, "loss": 1.7993, "theoretical_loss": 4.558503750807725, "tokens_seen": 147128320 }, { "epoch": 0.04, "learning_rate": 0.0007720441323971917, "loss": 1.7828, "theoretical_loss": 4.558239273967234, "tokens_seen": 147193856 }, { "epoch": 0.04, "learning_rate": 0.0007720280842527583, "loss": 1.8936, "theoretical_loss": 4.5579749478097025, "tokens_seen": 147259392 }, { "epoch": 0.04, "learning_rate": 0.0007720120361083251, "loss": 1.8044, "theoretical_loss": 4.557710772182278, "tokens_seen": 147324928 }, { "epoch": 0.04, "learning_rate": 0.0007719959879638917, "loss": 1.8211, "theoretical_loss": 4.5574467469323325, "tokens_seen": 147390464 }, { "epoch": 0.04, "objective/train/docs_used": 56056, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8968627452850342, "objective/train/theoretical_loss": 4.557182871907457, "objective/train/tokens_used": 167916000, "theoretical_loss": 4.557182871907457, "tokens_seen": 147456000 }, { "epoch": 0.04, "learning_rate": 0.0007719799398194585, "loss": 1.8249, "theoretical_loss": 4.557182871907457, "tokens_seen": 147456000 }, { "epoch": 0.04, "learning_rate": 0.0007719638916750251, "loss": 1.7435, "theoretical_loss": 4.556919146955469, "tokens_seen": 147521536 }, { "epoch": 0.04, "learning_rate": 0.0007719478435305919, "loss": 1.7664, "theoretical_loss": 4.556655571924404, "tokens_seen": 147587072 }, { "epoch": 0.04, "learning_rate": 0.0007719317953861585, "loss": 1.8627, "theoretical_loss": 4.556392146662521, "tokens_seen": 147652608 }, { "epoch": 0.04, "learning_rate": 0.0007719157472417253, "loss": 1.8259, "theoretical_loss": 4.556128871018297, "tokens_seen": 147718144 }, { "epoch": 0.04, "learning_rate": 0.0007718996990972919, "loss": 2.0033, "theoretical_loss": 4.555865744840434, "tokens_seen": 147783680 }, { "epoch": 0.04, "learning_rate": 0.0007718836509528587, "loss": 1.698, "theoretical_loss": 4.55560276797785, "tokens_seen": 147849216 }, { "epoch": 0.04, "learning_rate": 0.0007718676028084253, "loss": 1.7972, "theoretical_loss": 4.5553399402796835, "tokens_seen": 147914752 }, { "epoch": 0.04, "learning_rate": 0.0007718515546639921, "loss": 1.7489, "theoretical_loss": 4.5550772615952955, "tokens_seen": 147980288 }, { "epoch": 0.04, "learning_rate": 0.0007718355065195587, "loss": 1.8099, "theoretical_loss": 4.554814731774259, "tokens_seen": 148045824 }, { "epoch": 0.04, "learning_rate": 0.0007718194583751255, "loss": 1.8885, "theoretical_loss": 4.554552350666373, "tokens_seen": 148111360 }, { "epoch": 0.04, "learning_rate": 0.0007718034102306921, "loss": 1.8405, "theoretical_loss": 4.554290118121649, "tokens_seen": 148176896 }, { "epoch": 0.04, "learning_rate": 0.0007717873620862589, "loss": 1.8405, "theoretical_loss": 4.554028033990319, "tokens_seen": 148242432 }, { "epoch": 0.04, "learning_rate": 0.0007717713139418255, "loss": 1.81, "theoretical_loss": 4.5537660981228285, "tokens_seen": 148307968 }, { "epoch": 0.04, "learning_rate": 0.0007717552657973923, "loss": 1.8125, "theoretical_loss": 4.553504310369844, "tokens_seen": 148373504 }, { "epoch": 0.04, "learning_rate": 0.0007717392176529589, "loss": 1.8753, "theoretical_loss": 4.553242670582245, "tokens_seen": 148439040 }, { "epoch": 0.05, "learning_rate": 0.0007717231695085256, "loss": 1.9341, "theoretical_loss": 4.55298117861113, "tokens_seen": 148504576 }, { "epoch": 0.05, "learning_rate": 0.0007717071213640923, "loss": 1.7652, "theoretical_loss": 4.55271983430781, "tokens_seen": 148570112 }, { "epoch": 0.05, "learning_rate": 0.000771691073219659, "loss": 1.9011, "theoretical_loss": 4.552458637523815, "tokens_seen": 148635648 }, { "epoch": 0.05, "learning_rate": 0.0007716750250752257, "loss": 1.7245, "theoretical_loss": 4.552197588110882, "tokens_seen": 148701184 }, { "epoch": 0.05, "learning_rate": 0.0007716589769307924, "loss": 1.736, "theoretical_loss": 4.551936685920971, "tokens_seen": 148766720 }, { "epoch": 0.05, "learning_rate": 0.0007716429287863591, "loss": 1.6701, "theoretical_loss": 4.551675930806251, "tokens_seen": 148832256 }, { "epoch": 0.05, "learning_rate": 0.0007716268806419258, "loss": 1.8453, "theoretical_loss": 4.551415322619107, "tokens_seen": 148897792 }, { "epoch": 0.05, "learning_rate": 0.0007716108324974925, "loss": 1.8299, "theoretical_loss": 4.551154861212133, "tokens_seen": 148963328 }, { "epoch": 0.05, "learning_rate": 0.0007715947843530592, "loss": 1.97, "theoretical_loss": 4.550894546438139, "tokens_seen": 149028864 }, { "epoch": 0.05, "learning_rate": 0.0007715787362086258, "loss": 1.7303, "theoretical_loss": 4.550634378150148, "tokens_seen": 149094400 }, { "epoch": 0.05, "learning_rate": 0.0007715626880641926, "loss": 1.8074, "theoretical_loss": 4.550374356201392, "tokens_seen": 149159936 }, { "epoch": 0.05, "learning_rate": 0.0007715466399197592, "loss": 1.8377, "theoretical_loss": 4.550114480445316, "tokens_seen": 149225472 }, { "epoch": 0.05, "learning_rate": 0.000771530591775326, "loss": 1.8604, "theoretical_loss": 4.549854750735575, "tokens_seen": 149291008 }, { "epoch": 0.05, "learning_rate": 0.0007715145436308926, "loss": 1.9376, "theoretical_loss": 4.5495951669260375, "tokens_seen": 149356544 }, { "epoch": 0.05, "learning_rate": 0.0007714984954864594, "loss": 1.8481, "theoretical_loss": 4.54933572887078, "tokens_seen": 149422080 }, { "epoch": 0.05, "learning_rate": 0.000771482447342026, "loss": 1.8748, "theoretical_loss": 4.5490764364240865, "tokens_seen": 149487616 }, { "epoch": 0.05, "learning_rate": 0.0007714663991975928, "loss": 1.7571, "theoretical_loss": 4.548817289440455, "tokens_seen": 149553152 }, { "epoch": 0.05, "learning_rate": 0.0007714503510531594, "loss": 1.7867, "theoretical_loss": 4.548558287774592, "tokens_seen": 149618688 }, { "epoch": 0.05, "learning_rate": 0.0007714343029087262, "loss": 1.857, "theoretical_loss": 4.548299431281409, "tokens_seen": 149684224 }, { "epoch": 0.05, "learning_rate": 0.0007714182547642928, "loss": 1.773, "theoretical_loss": 4.548040719816029, "tokens_seen": 149749760 }, { "epoch": 0.05, "learning_rate": 0.0007714022066198596, "loss": 1.8777, "theoretical_loss": 4.547782153233783, "tokens_seen": 149815296 }, { "epoch": 0.05, "learning_rate": 0.0007713861584754262, "loss": 1.7381, "theoretical_loss": 4.547523731390206, "tokens_seen": 149880832 }, { "epoch": 0.05, "learning_rate": 0.000771370110330993, "loss": 1.7556, "theoretical_loss": 4.547265454141046, "tokens_seen": 149946368 }, { "epoch": 0.05, "learning_rate": 0.0007713540621865596, "loss": 1.7892, "theoretical_loss": 4.547007321342251, "tokens_seen": 150011904 }, { "epoch": 0.05, "learning_rate": 0.0007713380140421264, "loss": 1.8294, "theoretical_loss": 4.546749332849982, "tokens_seen": 150077440 }, { "epoch": 0.05, "learning_rate": 0.000771321965897693, "loss": 1.7976, "theoretical_loss": 4.546491488520601, "tokens_seen": 150142976 }, { "epoch": 0.05, "learning_rate": 0.0007713059177532598, "loss": 1.7639, "theoretical_loss": 4.546233788210679, "tokens_seen": 150208512 }, { "epoch": 0.05, "learning_rate": 0.0007712898696088264, "loss": 1.7777, "theoretical_loss": 4.545976231776988, "tokens_seen": 150274048 }, { "epoch": 0.05, "learning_rate": 0.0007712738214643932, "loss": 1.8345, "theoretical_loss": 4.545718819076509, "tokens_seen": 150339584 }, { "epoch": 0.05, "learning_rate": 0.0007712577733199598, "loss": 1.8569, "theoretical_loss": 4.545461549966426, "tokens_seen": 150405120 }, { "epoch": 0.05, "learning_rate": 0.0007712417251755266, "loss": 1.7723, "theoretical_loss": 4.545204424304126, "tokens_seen": 150470656 }, { "epoch": 0.05, "learning_rate": 0.0007712256770310932, "loss": 1.7852, "theoretical_loss": 4.544947441947201, "tokens_seen": 150536192 }, { "epoch": 0.05, "learning_rate": 0.00077120962888666, "loss": 1.7586, "theoretical_loss": 4.544690602753445, "tokens_seen": 150601728 }, { "epoch": 0.05, "learning_rate": 0.0007711935807422266, "loss": 1.8189, "theoretical_loss": 4.544433906580857, "tokens_seen": 150667264 }, { "epoch": 0.05, "objective/train/docs_used": 56915, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7465159893035889, "objective/train/theoretical_loss": 4.544177353287637, "objective/train/tokens_used": 171192800, "theoretical_loss": 4.544177353287637, "tokens_seen": 150732800 }, { "epoch": 0.05, "learning_rate": 0.0007711775325977934, "loss": 1.7764, "theoretical_loss": 4.544177353287637, "tokens_seen": 150732800 }, { "epoch": 0.05, "learning_rate": 0.0007711614844533602, "loss": 1.8765, "theoretical_loss": 4.543920942732187, "tokens_seen": 150798336 }, { "epoch": 0.05, "learning_rate": 0.0007711454363089268, "loss": 1.7625, "theoretical_loss": 4.543664674773112, "tokens_seen": 150863872 }, { "epoch": 0.05, "learning_rate": 0.0007711293881644936, "loss": 1.8125, "theoretical_loss": 4.543408549269218, "tokens_seen": 150929408 }, { "epoch": 0.05, "learning_rate": 0.0007711133400200602, "loss": 1.7914, "theoretical_loss": 4.54315256607951, "tokens_seen": 150994944 }, { "epoch": 0.05, "learning_rate": 0.000771097291875627, "loss": 1.8573, "theoretical_loss": 4.542896725063195, "tokens_seen": 151060480 }, { "epoch": 0.05, "learning_rate": 0.0007710812437311936, "loss": 1.6647, "theoretical_loss": 4.542641026079683, "tokens_seen": 151126016 }, { "epoch": 0.05, "learning_rate": 0.0007710651955867604, "loss": 1.8338, "theoretical_loss": 4.542385468988581, "tokens_seen": 151191552 }, { "epoch": 0.05, "learning_rate": 0.000771049147442327, "loss": 1.8271, "theoretical_loss": 4.542130053649695, "tokens_seen": 151257088 }, { "epoch": 0.05, "learning_rate": 0.0007710330992978938, "loss": 1.6977, "theoretical_loss": 4.541874779923031, "tokens_seen": 151322624 }, { "epoch": 0.05, "learning_rate": 0.0007710170511534604, "loss": 1.7345, "theoretical_loss": 4.541619647668796, "tokens_seen": 151388160 }, { "epoch": 0.05, "learning_rate": 0.0007710010030090272, "loss": 1.9003, "theoretical_loss": 4.541364656747392, "tokens_seen": 151453696 }, { "epoch": 0.05, "learning_rate": 0.0007709849548645938, "loss": 1.7489, "theoretical_loss": 4.541109807019422, "tokens_seen": 151519232 }, { "epoch": 0.05, "learning_rate": 0.0007709689067201606, "loss": 1.7527, "theoretical_loss": 4.540855098345683, "tokens_seen": 151584768 }, { "epoch": 0.05, "learning_rate": 0.0007709528585757272, "loss": 1.784, "theoretical_loss": 4.540600530587175, "tokens_seen": 151650304 }, { "epoch": 0.05, "learning_rate": 0.000770936810431294, "loss": 1.7235, "theoretical_loss": 4.5403461036050885, "tokens_seen": 151715840 }, { "epoch": 0.05, "learning_rate": 0.0007709207622868606, "loss": 1.838, "theoretical_loss": 4.540091817260815, "tokens_seen": 151781376 }, { "epoch": 0.05, "learning_rate": 0.0007709047141424274, "loss": 1.8029, "theoretical_loss": 4.539837671415941, "tokens_seen": 151846912 }, { "epoch": 0.05, "learning_rate": 0.000770888665997994, "loss": 1.7071, "theoretical_loss": 4.539583665932249, "tokens_seen": 151912448 }, { "epoch": 0.05, "learning_rate": 0.0007708726178535608, "loss": 1.7128, "theoretical_loss": 4.539329800671718, "tokens_seen": 151977984 }, { "epoch": 0.05, "learning_rate": 0.0007708565697091274, "loss": 1.7997, "theoretical_loss": 4.539076075496519, "tokens_seen": 152043520 }, { "epoch": 0.05, "learning_rate": 0.0007708405215646942, "loss": 1.9835, "theoretical_loss": 4.538822490269021, "tokens_seen": 152109056 }, { "epoch": 0.05, "learning_rate": 0.0007708244734202608, "loss": 1.7025, "theoretical_loss": 4.538569044851785, "tokens_seen": 152174592 }, { "epoch": 0.05, "learning_rate": 0.0007708084252758276, "loss": 1.7776, "theoretical_loss": 4.538315739107571, "tokens_seen": 152240128 }, { "epoch": 0.05, "learning_rate": 0.0007707923771313942, "loss": 1.63, "theoretical_loss": 4.5380625728993245, "tokens_seen": 152305664 }, { "epoch": 0.05, "learning_rate": 0.000770776328986961, "loss": 1.736, "theoretical_loss": 4.537809546090193, "tokens_seen": 152371200 }, { "epoch": 0.05, "learning_rate": 0.0007707602808425276, "loss": 1.8733, "theoretical_loss": 4.53755665854351, "tokens_seen": 152436736 }, { "epoch": 0.05, "learning_rate": 0.0007707442326980944, "loss": 1.73, "theoretical_loss": 4.537303910122807, "tokens_seen": 152502272 }, { "epoch": 0.05, "learning_rate": 0.000770728184553661, "loss": 1.753, "theoretical_loss": 4.537051300691805, "tokens_seen": 152567808 }, { "epoch": 0.05, "learning_rate": 0.0007707121364092278, "loss": 1.7725, "theoretical_loss": 4.5367988301144155, "tokens_seen": 152633344 }, { "epoch": 0.05, "learning_rate": 0.0007706960882647944, "loss": 1.7438, "theoretical_loss": 4.536546498254746, "tokens_seen": 152698880 }, { "epoch": 0.05, "learning_rate": 0.0007706800401203612, "loss": 1.8448, "theoretical_loss": 4.536294304977092, "tokens_seen": 152764416 }, { "epoch": 0.05, "learning_rate": 0.0007706639919759278, "loss": 1.76, "theoretical_loss": 4.536042250145941, "tokens_seen": 152829952 }, { "epoch": 0.05, "learning_rate": 0.0007706479438314946, "loss": 1.5725, "theoretical_loss": 4.53579033362597, "tokens_seen": 152895488 }, { "epoch": 0.05, "learning_rate": 0.0007706318956870612, "loss": 1.8505, "theoretical_loss": 4.535538555282048, "tokens_seen": 152961024 }, { "epoch": 0.05, "learning_rate": 0.000770615847542628, "loss": 1.9077, "theoretical_loss": 4.535286914979232, "tokens_seen": 153026560 }, { "epoch": 0.05, "learning_rate": 0.0007705997993981946, "loss": 1.7547, "theoretical_loss": 4.5350354125827685, "tokens_seen": 153092096 }, { "epoch": 0.05, "learning_rate": 0.0007705837512537614, "loss": 1.747, "theoretical_loss": 4.534784047958096, "tokens_seen": 153157632 }, { "epoch": 0.05, "learning_rate": 0.000770567703109328, "loss": 1.7225, "theoretical_loss": 4.534532820970839, "tokens_seen": 153223168 }, { "epoch": 0.05, "learning_rate": 0.0007705516549648948, "loss": 1.7695, "theoretical_loss": 4.534281731486812, "tokens_seen": 153288704 }, { "epoch": 0.05, "learning_rate": 0.0007705356068204614, "loss": 1.7913, "theoretical_loss": 4.5340307793720145, "tokens_seen": 153354240 }, { "epoch": 0.05, "learning_rate": 0.0007705195586760282, "loss": 1.703, "theoretical_loss": 4.533779964492638, "tokens_seen": 153419776 }, { "epoch": 0.05, "learning_rate": 0.0007705035105315948, "loss": 1.695, "theoretical_loss": 4.533529286715059, "tokens_seen": 153485312 }, { "epoch": 0.05, "learning_rate": 0.0007704874623871616, "loss": 1.7663, "theoretical_loss": 4.533278745905843, "tokens_seen": 153550848 }, { "epoch": 0.05, "learning_rate": 0.0007704714142427282, "loss": 1.6136, "theoretical_loss": 4.533028341931738, "tokens_seen": 153616384 }, { "epoch": 0.05, "learning_rate": 0.000770455366098295, "loss": 1.9402, "theoretical_loss": 4.532778074659685, "tokens_seen": 153681920 }, { "epoch": 0.05, "learning_rate": 0.0007704393179538616, "loss": 1.8058, "theoretical_loss": 4.532527943956804, "tokens_seen": 153747456 }, { "epoch": 0.05, "learning_rate": 0.0007704232698094284, "loss": 1.8964, "theoretical_loss": 4.532277949690407, "tokens_seen": 153812992 }, { "epoch": 0.05, "learning_rate": 0.000770407221664995, "loss": 1.6986, "theoretical_loss": 4.532028091727986, "tokens_seen": 153878528 }, { "epoch": 0.05, "learning_rate": 0.0007703911735205618, "loss": 1.6689, "theoretical_loss": 4.531778369937222, "tokens_seen": 153944064 }, { "epoch": 0.05, "objective/train/docs_used": 58041, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8966615200042725, "objective/train/theoretical_loss": 4.53152878418598, "objective/train/tokens_used": 174469600, "theoretical_loss": 4.53152878418598, "tokens_seen": 154009600 }, { "epoch": 0.05, "learning_rate": 0.0007703751253761284, "loss": 1.7864, "theoretical_loss": 4.53152878418598, "tokens_seen": 154009600 }, { "epoch": 0.05, "learning_rate": 0.0007703590772316952, "loss": 1.724, "theoretical_loss": 4.531279334342306, "tokens_seen": 154075136 }, { "epoch": 0.05, "learning_rate": 0.0007703430290872618, "loss": 1.7493, "theoretical_loss": 4.531030020274436, "tokens_seen": 154140672 }, { "epoch": 0.05, "learning_rate": 0.0007703269809428285, "loss": 1.7452, "theoretical_loss": 4.530780841850783, "tokens_seen": 154206208 }, { "epoch": 0.05, "learning_rate": 0.0007703109327983952, "loss": 1.7928, "theoretical_loss": 4.5305317989399505, "tokens_seen": 154271744 }, { "epoch": 0.05, "learning_rate": 0.0007702948846539619, "loss": 1.7797, "theoretical_loss": 4.530282891410717, "tokens_seen": 154337280 }, { "epoch": 0.05, "learning_rate": 0.0007702788365095286, "loss": 1.7927, "theoretical_loss": 4.530034119132052, "tokens_seen": 154402816 }, { "epoch": 0.05, "learning_rate": 0.0007702627883650953, "loss": 1.7336, "theoretical_loss": 4.529785481973102, "tokens_seen": 154468352 }, { "epoch": 0.05, "learning_rate": 0.000770246740220662, "loss": 1.7642, "theoretical_loss": 4.529536979803195, "tokens_seen": 154533888 }, { "epoch": 0.05, "learning_rate": 0.0007702306920762287, "loss": 1.9063, "theoretical_loss": 4.529288612491846, "tokens_seen": 154599424 }, { "epoch": 0.05, "learning_rate": 0.0007702146439317954, "loss": 1.6956, "theoretical_loss": 4.5290403799087455, "tokens_seen": 154664960 }, { "epoch": 0.05, "learning_rate": 0.0007701985957873621, "loss": 1.9776, "theoretical_loss": 4.528792281923769, "tokens_seen": 154730496 }, { "epoch": 0.05, "learning_rate": 0.0007701825476429287, "loss": 1.8414, "theoretical_loss": 4.52854431840697, "tokens_seen": 154796032 }, { "epoch": 0.05, "learning_rate": 0.0007701664994984955, "loss": 1.8142, "theoretical_loss": 4.528296489228585, "tokens_seen": 154861568 }, { "epoch": 0.05, "learning_rate": 0.0007701504513540621, "loss": 1.6601, "theoretical_loss": 4.528048794259028, "tokens_seen": 154927104 }, { "epoch": 0.05, "learning_rate": 0.0007701344032096289, "loss": 1.8983, "theoretical_loss": 4.527801233368896, "tokens_seen": 154992640 }, { "epoch": 0.05, "learning_rate": 0.0007701183550651955, "loss": 1.7475, "theoretical_loss": 4.527553806428962, "tokens_seen": 155058176 }, { "epoch": 0.05, "learning_rate": 0.0007701023069207623, "loss": 1.8392, "theoretical_loss": 4.5273065133101795, "tokens_seen": 155123712 }, { "epoch": 0.05, "learning_rate": 0.0007700862587763289, "loss": 1.7844, "theoretical_loss": 4.527059353883682, "tokens_seen": 155189248 }, { "epoch": 0.05, "learning_rate": 0.0007700702106318957, "loss": 1.8887, "theoretical_loss": 4.526812328020778, "tokens_seen": 155254784 }, { "epoch": 0.05, "learning_rate": 0.0007700541624874623, "loss": 1.736, "theoretical_loss": 4.52656543559296, "tokens_seen": 155320320 }, { "epoch": 0.05, "learning_rate": 0.0007700381143430291, "loss": 1.7517, "theoretical_loss": 4.526318676471891, "tokens_seen": 155385856 }, { "epoch": 0.05, "learning_rate": 0.0007700220661985957, "loss": 1.7906, "theoretical_loss": 4.526072050529417, "tokens_seen": 155451392 }, { "epoch": 0.05, "learning_rate": 0.0007700060180541625, "loss": 1.8339, "theoretical_loss": 4.52582555763756, "tokens_seen": 155516928 }, { "epoch": 0.05, "learning_rate": 0.0007699899699097291, "loss": 1.8402, "theoretical_loss": 4.525579197668517, "tokens_seen": 155582464 }, { "epoch": 0.05, "learning_rate": 0.0007699739217652959, "loss": 1.8406, "theoretical_loss": 4.525332970494663, "tokens_seen": 155648000 }, { "epoch": 0.05, "learning_rate": 0.0007699578736208627, "loss": 1.793, "theoretical_loss": 4.525086875988549, "tokens_seen": 155713536 }, { "epoch": 0.05, "learning_rate": 0.0007699418254764293, "loss": 1.8599, "theoretical_loss": 4.524840914022902, "tokens_seen": 155779072 }, { "epoch": 0.05, "learning_rate": 0.000769925777331996, "loss": 1.7589, "theoretical_loss": 4.524595084470625, "tokens_seen": 155844608 }, { "epoch": 0.05, "learning_rate": 0.0007699097291875627, "loss": 1.8455, "theoretical_loss": 4.524349387204794, "tokens_seen": 155910144 }, { "epoch": 0.05, "learning_rate": 0.0007698936810431295, "loss": 1.6923, "theoretical_loss": 4.524103822098664, "tokens_seen": 155975680 }, { "epoch": 0.05, "learning_rate": 0.0007698776328986961, "loss": 1.7622, "theoretical_loss": 4.52385838902566, "tokens_seen": 156041216 }, { "epoch": 0.05, "learning_rate": 0.0007698615847542629, "loss": 1.6998, "theoretical_loss": 4.523613087859386, "tokens_seen": 156106752 }, { "epoch": 0.05, "learning_rate": 0.0007698455366098295, "loss": 1.7923, "theoretical_loss": 4.523367918473614, "tokens_seen": 156172288 }, { "epoch": 0.05, "learning_rate": 0.0007698294884653963, "loss": 1.814, "theoretical_loss": 4.523122880742298, "tokens_seen": 156237824 }, { "epoch": 0.05, "learning_rate": 0.0007698134403209629, "loss": 1.7696, "theoretical_loss": 4.522877974539556, "tokens_seen": 156303360 }, { "epoch": 0.05, "learning_rate": 0.0007697973921765297, "loss": 1.8467, "theoretical_loss": 4.522633199739687, "tokens_seen": 156368896 }, { "epoch": 0.05, "learning_rate": 0.0007697813440320963, "loss": 1.7256, "theoretical_loss": 4.522388556217157, "tokens_seen": 156434432 }, { "epoch": 0.05, "learning_rate": 0.0007697652958876631, "loss": 1.7527, "theoretical_loss": 4.522144043846609, "tokens_seen": 156499968 }, { "epoch": 0.05, "learning_rate": 0.0007697492477432297, "loss": 1.784, "theoretical_loss": 4.521899662502856, "tokens_seen": 156565504 }, { "epoch": 0.05, "learning_rate": 0.0007697331995987965, "loss": 1.7029, "theoretical_loss": 4.5216554120608805, "tokens_seen": 156631040 }, { "epoch": 0.05, "learning_rate": 0.0007697171514543631, "loss": 1.7115, "theoretical_loss": 4.521411292395841, "tokens_seen": 156696576 }, { "epoch": 0.05, "learning_rate": 0.0007697011033099299, "loss": 1.7872, "theoretical_loss": 4.521167303383065, "tokens_seen": 156762112 }, { "epoch": 0.05, "learning_rate": 0.0007696850551654965, "loss": 1.6923, "theoretical_loss": 4.520923444898051, "tokens_seen": 156827648 }, { "epoch": 0.05, "learning_rate": 0.0007696690070210633, "loss": 1.7266, "theoretical_loss": 4.520679716816467, "tokens_seen": 156893184 }, { "epoch": 0.05, "learning_rate": 0.0007696529588766299, "loss": 1.7961, "theoretical_loss": 4.520436119014155, "tokens_seen": 156958720 }, { "epoch": 0.05, "learning_rate": 0.0007696369107321967, "loss": 1.7192, "theoretical_loss": 4.520192651367121, "tokens_seen": 157024256 }, { "epoch": 0.05, "learning_rate": 0.0007696208625877633, "loss": 1.5909, "theoretical_loss": 4.519949313751547, "tokens_seen": 157089792 }, { "epoch": 0.05, "learning_rate": 0.0007696048144433301, "loss": 1.8818, "theoretical_loss": 4.519706106043779, "tokens_seen": 157155328 }, { "epoch": 0.05, "learning_rate": 0.0007695887662988967, "loss": 1.7198, "theoretical_loss": 4.519463028120338, "tokens_seen": 157220864 }, { "epoch": 0.05, "objective/train/docs_used": 59100, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.76185941696167, "objective/train/theoretical_loss": 4.519220079857908, "objective/train/tokens_used": 177746400, "theoretical_loss": 4.519220079857908, "tokens_seen": 157286400 }, { "epoch": 0.05, "learning_rate": 0.0007695727181544635, "loss": 1.7576, "theoretical_loss": 4.519220079857908, "tokens_seen": 157286400 }, { "epoch": 0.05, "learning_rate": 0.0007695566700100301, "loss": 1.7981, "theoretical_loss": 4.518977261133343, "tokens_seen": 157351936 }, { "epoch": 0.05, "learning_rate": 0.0007695406218655969, "loss": 1.6934, "theoretical_loss": 4.518734571823668, "tokens_seen": 157417472 }, { "epoch": 0.05, "learning_rate": 0.0007695245737211635, "loss": 1.7905, "theoretical_loss": 4.518492011806073, "tokens_seen": 157483008 }, { "epoch": 0.05, "learning_rate": 0.0007695085255767303, "loss": 1.7677, "theoretical_loss": 4.518249580957917, "tokens_seen": 157548544 }, { "epoch": 0.05, "learning_rate": 0.0007694924774322969, "loss": 1.8178, "theoretical_loss": 4.518007279156725, "tokens_seen": 157614080 }, { "epoch": 0.05, "learning_rate": 0.0007694764292878637, "loss": 1.8327, "theoretical_loss": 4.5177651062801925, "tokens_seen": 157679616 }, { "epoch": 0.05, "learning_rate": 0.0007694603811434303, "loss": 1.8089, "theoretical_loss": 4.517523062206175, "tokens_seen": 157745152 }, { "epoch": 0.05, "learning_rate": 0.0007694443329989971, "loss": 1.7748, "theoretical_loss": 4.517281146812701, "tokens_seen": 157810688 }, { "epoch": 0.05, "learning_rate": 0.0007694282848545637, "loss": 1.8415, "theoretical_loss": 4.517039359977961, "tokens_seen": 157876224 }, { "epoch": 0.05, "learning_rate": 0.0007694122367101305, "loss": 1.8614, "theoretical_loss": 4.516797701580314, "tokens_seen": 157941760 }, { "epoch": 0.05, "learning_rate": 0.0007693961885656971, "loss": 1.6887, "theoretical_loss": 4.516556171498284, "tokens_seen": 158007296 }, { "epoch": 0.05, "learning_rate": 0.0007693801404212639, "loss": 1.6983, "theoretical_loss": 4.516314769610557, "tokens_seen": 158072832 }, { "epoch": 0.05, "learning_rate": 0.0007693640922768305, "loss": 1.693, "theoretical_loss": 4.516073495795988, "tokens_seen": 158138368 }, { "epoch": 0.05, "learning_rate": 0.0007693480441323973, "loss": 1.8146, "theoretical_loss": 4.5158323499335955, "tokens_seen": 158203904 }, { "epoch": 0.05, "learning_rate": 0.0007693319959879639, "loss": 1.7686, "theoretical_loss": 4.515591331902561, "tokens_seen": 158269440 }, { "epoch": 0.05, "learning_rate": 0.0007693159478435307, "loss": 1.6949, "theoretical_loss": 4.515350441582231, "tokens_seen": 158334976 }, { "epoch": 0.05, "learning_rate": 0.0007692998996990973, "loss": 1.8541, "theoretical_loss": 4.515109678852117, "tokens_seen": 158400512 }, { "epoch": 0.05, "learning_rate": 0.0007692838515546641, "loss": 1.9401, "theoretical_loss": 4.514869043591891, "tokens_seen": 158466048 }, { "epoch": 0.05, "learning_rate": 0.0007692678034102307, "loss": 1.7639, "theoretical_loss": 4.514628535681391, "tokens_seen": 158531584 }, { "epoch": 0.05, "learning_rate": 0.0007692517552657975, "loss": 1.7224, "theoretical_loss": 4.514388155000615, "tokens_seen": 158597120 }, { "epoch": 0.05, "learning_rate": 0.0007692357071213641, "loss": 1.76, "theoretical_loss": 4.5141479014297285, "tokens_seen": 158662656 }, { "epoch": 0.05, "learning_rate": 0.0007692196589769309, "loss": 1.8276, "theoretical_loss": 4.513907774849054, "tokens_seen": 158728192 }, { "epoch": 0.05, "learning_rate": 0.0007692036108324975, "loss": 1.8055, "theoretical_loss": 4.513667775139078, "tokens_seen": 158793728 }, { "epoch": 0.05, "learning_rate": 0.0007691875626880643, "loss": 1.7416, "theoretical_loss": 4.5134279021804495, "tokens_seen": 158859264 }, { "epoch": 0.05, "learning_rate": 0.0007691715145436309, "loss": 1.675, "theoretical_loss": 4.513188155853978, "tokens_seen": 158924800 }, { "epoch": 0.05, "learning_rate": 0.0007691554663991977, "loss": 1.8391, "theoretical_loss": 4.512948536040636, "tokens_seen": 158990336 }, { "epoch": 0.05, "learning_rate": 0.0007691394182547643, "loss": 1.8972, "theoretical_loss": 4.512709042621554, "tokens_seen": 159055872 }, { "epoch": 0.05, "learning_rate": 0.0007691233701103311, "loss": 1.7549, "theoretical_loss": 4.512469675478026, "tokens_seen": 159121408 }, { "epoch": 0.05, "learning_rate": 0.0007691073219658977, "loss": 1.7728, "theoretical_loss": 4.512230434491503, "tokens_seen": 159186944 }, { "epoch": 0.05, "learning_rate": 0.0007690912738214645, "loss": 1.8985, "theoretical_loss": 4.511991319543599, "tokens_seen": 159252480 }, { "epoch": 0.05, "learning_rate": 0.0007690752256770311, "loss": 1.7964, "theoretical_loss": 4.511752330516086, "tokens_seen": 159318016 }, { "epoch": 0.05, "learning_rate": 0.0007690591775325979, "loss": 1.658, "theoretical_loss": 4.5115134672908965, "tokens_seen": 159383552 }, { "epoch": 0.05, "learning_rate": 0.0007690431293881645, "loss": 1.9224, "theoretical_loss": 4.511274729750123, "tokens_seen": 159449088 }, { "epoch": 0.05, "learning_rate": 0.0007690270812437313, "loss": 1.7319, "theoretical_loss": 4.5110361177760145, "tokens_seen": 159514624 }, { "epoch": 0.05, "learning_rate": 0.0007690110330992979, "loss": 1.7229, "theoretical_loss": 4.5107976312509805, "tokens_seen": 159580160 }, { "epoch": 0.05, "learning_rate": 0.0007689949849548647, "loss": 1.7498, "theoretical_loss": 4.510559270057587, "tokens_seen": 159645696 }, { "epoch": 0.05, "learning_rate": 0.0007689789368104313, "loss": 1.7295, "theoretical_loss": 4.51032103407856, "tokens_seen": 159711232 }, { "epoch": 0.05, "learning_rate": 0.0007689628886659981, "loss": 1.7314, "theoretical_loss": 4.510082923196784, "tokens_seen": 159776768 }, { "epoch": 0.05, "learning_rate": 0.0007689468405215647, "loss": 1.7616, "theoretical_loss": 4.5098449372952985, "tokens_seen": 159842304 }, { "epoch": 0.05, "learning_rate": 0.0007689307923771315, "loss": 1.7165, "theoretical_loss": 4.5096070762573, "tokens_seen": 159907840 }, { "epoch": 0.05, "learning_rate": 0.0007689147442326981, "loss": 1.7341, "theoretical_loss": 4.509369339966146, "tokens_seen": 159973376 }, { "epoch": 0.05, "learning_rate": 0.0007688986960882648, "loss": 1.7801, "theoretical_loss": 4.509131728305347, "tokens_seen": 160038912 }, { "epoch": 0.05, "learning_rate": 0.0007688826479438315, "loss": 1.9064, "theoretical_loss": 4.50889424115857, "tokens_seen": 160104448 }, { "epoch": 0.05, "learning_rate": 0.0007688665997993982, "loss": 1.6784, "theoretical_loss": 4.50865687840964, "tokens_seen": 160169984 }, { "epoch": 0.05, "learning_rate": 0.0007688505516549649, "loss": 1.7526, "theoretical_loss": 4.508419639942538, "tokens_seen": 160235520 }, { "epoch": 0.05, "learning_rate": 0.0007688345035105316, "loss": 1.7459, "theoretical_loss": 4.5081825256413985, "tokens_seen": 160301056 }, { "epoch": 0.05, "learning_rate": 0.0007688184553660983, "loss": 1.8808, "theoretical_loss": 4.507945535390512, "tokens_seen": 160366592 }, { "epoch": 0.05, "learning_rate": 0.000768802407221665, "loss": 1.816, "theoretical_loss": 4.507708669074326, "tokens_seen": 160432128 }, { "epoch": 0.05, "learning_rate": 0.0007687863590772316, "loss": 1.7083, "theoretical_loss": 4.5074719265774394, "tokens_seen": 160497664 }, { "epoch": 0.05, "objective/train/docs_used": 60314, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.1015968322753906, "objective/train/theoretical_loss": 4.507235307784608, "objective/train/tokens_used": 181023200, "theoretical_loss": 4.507235307784608, "tokens_seen": 160563200 }, { "epoch": 0.05, "learning_rate": 0.0007687703109327984, "loss": 1.9357, "theoretical_loss": 4.507235307784608, "tokens_seen": 160563200 }, { "epoch": 0.05, "learning_rate": 0.0007687542627883651, "loss": 1.6547, "theoretical_loss": 4.5069988125807425, "tokens_seen": 160628736 }, { "epoch": 0.05, "learning_rate": 0.0007687382146439318, "loss": 1.7709, "theoretical_loss": 4.506762440850906, "tokens_seen": 160694272 }, { "epoch": 0.05, "learning_rate": 0.0007687221664994985, "loss": 1.7327, "theoretical_loss": 4.506526192480315, "tokens_seen": 160759808 }, { "epoch": 0.05, "learning_rate": 0.0007687061183550652, "loss": 1.7753, "theoretical_loss": 4.506290067354341, "tokens_seen": 160825344 }, { "epoch": 0.05, "learning_rate": 0.000768690070210632, "loss": 1.7335, "theoretical_loss": 4.506054065358508, "tokens_seen": 160890880 }, { "epoch": 0.05, "learning_rate": 0.0007686740220661986, "loss": 1.7166, "theoretical_loss": 4.505818186378491, "tokens_seen": 160956416 }, { "epoch": 0.05, "learning_rate": 0.0007686579739217653, "loss": 1.6904, "theoretical_loss": 4.505582430300121, "tokens_seen": 161021952 }, { "epoch": 0.05, "learning_rate": 0.000768641925777332, "loss": 1.8413, "theoretical_loss": 4.505346797009381, "tokens_seen": 161087488 }, { "epoch": 0.05, "learning_rate": 0.0007686258776328987, "loss": 1.7868, "theoretical_loss": 4.505111286392402, "tokens_seen": 161153024 }, { "epoch": 0.05, "learning_rate": 0.0007686098294884654, "loss": 1.6634, "theoretical_loss": 4.50487589833547, "tokens_seen": 161218560 }, { "epoch": 0.05, "learning_rate": 0.0007685937813440322, "loss": 1.6353, "theoretical_loss": 4.504640632725025, "tokens_seen": 161284096 }, { "epoch": 0.05, "learning_rate": 0.0007685777331995988, "loss": 1.7892, "theoretical_loss": 4.504405489447654, "tokens_seen": 161349632 }, { "epoch": 0.05, "learning_rate": 0.0007685616850551656, "loss": 1.7234, "theoretical_loss": 4.504170468390096, "tokens_seen": 161415168 }, { "epoch": 0.05, "learning_rate": 0.0007685456369107322, "loss": 1.6762, "theoretical_loss": 4.503935569439242, "tokens_seen": 161480704 }, { "epoch": 0.05, "learning_rate": 0.000768529588766299, "loss": 1.8172, "theoretical_loss": 4.5037007924821335, "tokens_seen": 161546240 }, { "epoch": 0.05, "learning_rate": 0.0007685135406218656, "loss": 1.7786, "theoretical_loss": 4.503466137405961, "tokens_seen": 161611776 }, { "epoch": 0.05, "learning_rate": 0.0007684974924774324, "loss": 1.732, "theoretical_loss": 4.503231604098065, "tokens_seen": 161677312 }, { "epoch": 0.05, "learning_rate": 0.000768481444332999, "loss": 1.8209, "theoretical_loss": 4.50299719244594, "tokens_seen": 161742848 }, { "epoch": 0.05, "learning_rate": 0.0007684653961885658, "loss": 1.8043, "theoretical_loss": 4.502762902337222, "tokens_seen": 161808384 }, { "epoch": 0.05, "learning_rate": 0.0007684493480441324, "loss": 1.8455, "theoretical_loss": 4.502528733659704, "tokens_seen": 161873920 }, { "epoch": 0.05, "learning_rate": 0.0007684332998996992, "loss": 1.7825, "theoretical_loss": 4.502294686301323, "tokens_seen": 161939456 }, { "epoch": 0.05, "learning_rate": 0.0007684172517552658, "loss": 1.6838, "theoretical_loss": 4.502060760150167, "tokens_seen": 162004992 }, { "epoch": 0.05, "learning_rate": 0.0007684012036108326, "loss": 1.8061, "theoretical_loss": 4.501826955094472, "tokens_seen": 162070528 }, { "epoch": 0.05, "learning_rate": 0.0007683851554663992, "loss": 1.6355, "theoretical_loss": 4.501593271022623, "tokens_seen": 162136064 }, { "epoch": 0.05, "learning_rate": 0.000768369107321966, "loss": 1.7908, "theoretical_loss": 4.501359707823149, "tokens_seen": 162201600 }, { "epoch": 0.05, "learning_rate": 0.0007683530591775326, "loss": 1.7182, "theoretical_loss": 4.501126265384732, "tokens_seen": 162267136 }, { "epoch": 0.05, "learning_rate": 0.0007683370110330994, "loss": 1.9053, "theoretical_loss": 4.500892943596199, "tokens_seen": 162332672 }, { "epoch": 0.05, "learning_rate": 0.000768320962888666, "loss": 1.8499, "theoretical_loss": 4.500659742346524, "tokens_seen": 162398208 }, { "epoch": 0.05, "learning_rate": 0.0007683049147442328, "loss": 1.7459, "theoretical_loss": 4.500426661524828, "tokens_seen": 162463744 }, { "epoch": 0.05, "learning_rate": 0.0007682888665997994, "loss": 1.774, "theoretical_loss": 4.500193701020379, "tokens_seen": 162529280 }, { "epoch": 0.05, "learning_rate": 0.0007682728184553662, "loss": 1.8263, "theoretical_loss": 4.499960860722592, "tokens_seen": 162594816 }, { "epoch": 0.05, "learning_rate": 0.0007682567703109328, "loss": 1.7356, "theoretical_loss": 4.499728140521025, "tokens_seen": 162660352 }, { "epoch": 0.05, "learning_rate": 0.0007682407221664996, "loss": 1.8387, "theoretical_loss": 4.499495540305388, "tokens_seen": 162725888 }, { "epoch": 0.05, "learning_rate": 0.0007682246740220662, "loss": 1.6331, "theoretical_loss": 4.49926305996553, "tokens_seen": 162791424 }, { "epoch": 0.05, "learning_rate": 0.000768208625877633, "loss": 1.7426, "theoretical_loss": 4.499030699391449, "tokens_seen": 162856960 }, { "epoch": 0.05, "learning_rate": 0.0007681925777331996, "loss": 1.617, "theoretical_loss": 4.49879845847329, "tokens_seen": 162922496 }, { "epoch": 0.05, "learning_rate": 0.0007681765295887664, "loss": 1.7124, "theoretical_loss": 4.498566337101337, "tokens_seen": 162988032 }, { "epoch": 0.05, "learning_rate": 0.000768160481444333, "loss": 1.7712, "theoretical_loss": 4.4983343351660245, "tokens_seen": 163053568 }, { "epoch": 0.05, "learning_rate": 0.0007681444332998998, "loss": 1.9108, "theoretical_loss": 4.498102452557928, "tokens_seen": 163119104 }, { "epoch": 0.05, "learning_rate": 0.0007681283851554664, "loss": 1.6287, "theoretical_loss": 4.497870689167771, "tokens_seen": 163184640 }, { "epoch": 0.05, "learning_rate": 0.0007681123370110332, "loss": 1.8195, "theoretical_loss": 4.497639044886412, "tokens_seen": 163250176 }, { "epoch": 0.05, "learning_rate": 0.0007680962888665998, "loss": 1.7055, "theoretical_loss": 4.497407519604866, "tokens_seen": 163315712 }, { "epoch": 0.05, "learning_rate": 0.0007680802407221666, "loss": 1.8599, "theoretical_loss": 4.49717611321428, "tokens_seen": 163381248 }, { "epoch": 0.05, "learning_rate": 0.0007680641925777332, "loss": 1.6922, "theoretical_loss": 4.496944825605951, "tokens_seen": 163446784 }, { "epoch": 0.05, "learning_rate": 0.0007680481444333, "loss": 1.6333, "theoretical_loss": 4.496713656671317, "tokens_seen": 163512320 }, { "epoch": 0.05, "learning_rate": 0.0007680320962888666, "loss": 1.8145, "theoretical_loss": 4.496482606301957, "tokens_seen": 163577856 }, { "epoch": 0.05, "learning_rate": 0.0007680160481444334, "loss": 1.7066, "theoretical_loss": 4.496251674389596, "tokens_seen": 163643392 }, { "epoch": 0.05, "learning_rate": 0.000768, "loss": 1.7063, "theoretical_loss": 4.496020860826098, "tokens_seen": 163708928 }, { "epoch": 0.05, "learning_rate": 0.0007679839518555668, "loss": 1.7411, "theoretical_loss": 4.49579016550347, "tokens_seen": 163774464 }, { "epoch": 0.05, "objective/train/docs_used": 61072, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7547411918640137, "objective/train/theoretical_loss": 4.49555958831386, "objective/train/tokens_used": 184300000, "theoretical_loss": 4.49555958831386, "tokens_seen": 163840000 }, { "epoch": 0.05, "learning_rate": 0.0007679679037111334, "loss": 1.7758, "theoretical_loss": 4.49555958831386, "tokens_seen": 163840000 }, { "epoch": 0.05, "learning_rate": 0.0007679518555667002, "loss": 1.7571, "theoretical_loss": 4.49532912914956, "tokens_seen": 163905536 }, { "epoch": 0.05, "learning_rate": 0.0007679358074222668, "loss": 1.7834, "theoretical_loss": 4.495098787903, "tokens_seen": 163971072 }, { "epoch": 0.05, "learning_rate": 0.0007679197592778336, "loss": 1.7879, "theoretical_loss": 4.4948685644667545, "tokens_seen": 164036608 }, { "epoch": 0.05, "learning_rate": 0.0007679037111334002, "loss": 1.6408, "theoretical_loss": 4.494638458733535, "tokens_seen": 164102144 }, { "epoch": 0.05, "learning_rate": 0.000767887662988967, "loss": 1.6375, "theoretical_loss": 4.494408470596195, "tokens_seen": 164167680 }, { "epoch": 0.05, "learning_rate": 0.0007678716148445336, "loss": 1.7134, "theoretical_loss": 4.494178599947729, "tokens_seen": 164233216 }, { "epoch": 0.05, "learning_rate": 0.0007678555667001004, "loss": 1.8468, "theoretical_loss": 4.4939488466812705, "tokens_seen": 164298752 }, { "epoch": 0.05, "learning_rate": 0.000767839518555667, "loss": 1.8571, "theoretical_loss": 4.493719210690093, "tokens_seen": 164364288 }, { "epoch": 0.05, "learning_rate": 0.0007678234704112338, "loss": 1.711, "theoretical_loss": 4.493489691867612, "tokens_seen": 164429824 }, { "epoch": 0.05, "learning_rate": 0.0007678074222668004, "loss": 1.6744, "theoretical_loss": 4.493260290107379, "tokens_seen": 164495360 }, { "epoch": 0.05, "learning_rate": 0.0007677913741223672, "loss": 1.6627, "theoretical_loss": 4.493031005303084, "tokens_seen": 164560896 }, { "epoch": 0.05, "learning_rate": 0.0007677753259779338, "loss": 1.7317, "theoretical_loss": 4.49280183734856, "tokens_seen": 164626432 }, { "epoch": 0.05, "learning_rate": 0.0007677592778335006, "loss": 1.8705, "theoretical_loss": 4.492572786137774, "tokens_seen": 164691968 }, { "epoch": 0.05, "learning_rate": 0.0007677432296890672, "loss": 1.7851, "theoretical_loss": 4.492343851564835, "tokens_seen": 164757504 }, { "epoch": 0.05, "learning_rate": 0.000767727181544634, "loss": 1.8575, "theoretical_loss": 4.492115033523987, "tokens_seen": 164823040 }, { "epoch": 0.05, "learning_rate": 0.0007677111334002006, "loss": 1.7289, "theoretical_loss": 4.491886331909615, "tokens_seen": 164888576 }, { "epoch": 0.05, "learning_rate": 0.0007676950852557674, "loss": 1.7442, "theoretical_loss": 4.491657746616241, "tokens_seen": 164954112 }, { "epoch": 0.05, "learning_rate": 0.000767679037111334, "loss": 1.6633, "theoretical_loss": 4.49142927753852, "tokens_seen": 165019648 }, { "epoch": 0.05, "learning_rate": 0.0007676629889669008, "loss": 1.6377, "theoretical_loss": 4.491200924571251, "tokens_seen": 165085184 }, { "epoch": 0.05, "learning_rate": 0.0007676469408224674, "loss": 1.5454, "theoretical_loss": 4.490972687609364, "tokens_seen": 165150720 }, { "epoch": 0.05, "learning_rate": 0.0007676308926780342, "loss": 1.7009, "theoretical_loss": 4.4907445665479315, "tokens_seen": 165216256 }, { "epoch": 0.05, "learning_rate": 0.0007676148445336008, "loss": 1.7974, "theoretical_loss": 4.490516561282157, "tokens_seen": 165281792 }, { "epoch": 0.05, "learning_rate": 0.0007675987963891676, "loss": 1.6644, "theoretical_loss": 4.4902886717073835, "tokens_seen": 165347328 }, { "epoch": 0.05, "learning_rate": 0.0007675827482447342, "loss": 1.7853, "theoretical_loss": 4.49006089771909, "tokens_seen": 165412864 }, { "epoch": 0.05, "learning_rate": 0.000767566700100301, "loss": 1.5877, "theoretical_loss": 4.489833239212888, "tokens_seen": 165478400 }, { "epoch": 0.05, "learning_rate": 0.0007675506519558676, "loss": 1.6106, "theoretical_loss": 4.48960569608453, "tokens_seen": 165543936 }, { "epoch": 0.05, "learning_rate": 0.0007675346038114344, "loss": 1.7594, "theoretical_loss": 4.489378268229897, "tokens_seen": 165609472 }, { "epoch": 0.05, "learning_rate": 0.000767518555667001, "loss": 1.8262, "theoretical_loss": 4.489150955545012, "tokens_seen": 165675008 }, { "epoch": 0.05, "learning_rate": 0.0007675025075225677, "loss": 1.7828, "theoretical_loss": 4.488923757926031, "tokens_seen": 165740544 }, { "epoch": 0.05, "learning_rate": 0.0007674864593781344, "loss": 1.6994, "theoretical_loss": 4.4886966752692405, "tokens_seen": 165806080 }, { "epoch": 0.05, "learning_rate": 0.0007674704112337011, "loss": 1.6913, "theoretical_loss": 4.488469707471066, "tokens_seen": 165871616 }, { "epoch": 0.05, "learning_rate": 0.0007674543630892678, "loss": 1.6736, "theoretical_loss": 4.488242854428064, "tokens_seen": 165937152 }, { "epoch": 0.05, "learning_rate": 0.0007674383149448345, "loss": 1.6912, "theoretical_loss": 4.488016116036929, "tokens_seen": 166002688 }, { "epoch": 0.05, "learning_rate": 0.0007674222668004012, "loss": 1.9221, "theoretical_loss": 4.487789492194485, "tokens_seen": 166068224 }, { "epoch": 0.05, "learning_rate": 0.0007674062186559679, "loss": 1.7546, "theoretical_loss": 4.487562982797693, "tokens_seen": 166133760 }, { "epoch": 0.05, "learning_rate": 0.0007673901705115346, "loss": 1.7376, "theoretical_loss": 4.487336587743643, "tokens_seen": 166199296 }, { "epoch": 0.05, "learning_rate": 0.0007673741223671013, "loss": 1.7413, "theoretical_loss": 4.487110306929562, "tokens_seen": 166264832 }, { "epoch": 0.05, "learning_rate": 0.000767358074222668, "loss": 1.7142, "theoretical_loss": 4.48688414025281, "tokens_seen": 166330368 }, { "epoch": 0.05, "learning_rate": 0.0007673420260782347, "loss": 1.8665, "theoretical_loss": 4.486658087610875, "tokens_seen": 166395904 }, { "epoch": 0.05, "learning_rate": 0.0007673259779338014, "loss": 1.6341, "theoretical_loss": 4.486432148901384, "tokens_seen": 166461440 }, { "epoch": 0.05, "learning_rate": 0.0007673099297893681, "loss": 1.7621, "theoretical_loss": 4.48620632402209, "tokens_seen": 166526976 }, { "epoch": 0.05, "learning_rate": 0.0007672938816449348, "loss": 1.7723, "theoretical_loss": 4.485980612870883, "tokens_seen": 166592512 }, { "epoch": 0.05, "learning_rate": 0.0007672778335005015, "loss": 1.7573, "theoretical_loss": 4.48575501534578, "tokens_seen": 166658048 }, { "epoch": 0.05, "learning_rate": 0.0007672617853560682, "loss": 1.7641, "theoretical_loss": 4.485529531344932, "tokens_seen": 166723584 }, { "epoch": 0.05, "learning_rate": 0.0007672457372116349, "loss": 1.8136, "theoretical_loss": 4.485304160766624, "tokens_seen": 166789120 }, { "epoch": 0.05, "learning_rate": 0.0007672296890672016, "loss": 1.7361, "theoretical_loss": 4.485078903509266, "tokens_seen": 166854656 }, { "epoch": 0.05, "learning_rate": 0.0007672136409227683, "loss": 1.8344, "theoretical_loss": 4.484853759471403, "tokens_seen": 166920192 }, { "epoch": 0.05, "learning_rate": 0.000767197592778335, "loss": 1.833, "theoretical_loss": 4.484628728551711, "tokens_seen": 166985728 }, { "epoch": 0.05, "learning_rate": 0.0007671815446339017, "loss": 1.7914, "theoretical_loss": 4.484403810648994, "tokens_seen": 167051264 }, { "epoch": 0.05, "objective/train/docs_used": 62380, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6307178735733032, "objective/train/theoretical_loss": 4.484179005662186, "objective/train/tokens_used": 187576800, "theoretical_loss": 4.484179005662186, "tokens_seen": 167116800 }, { "epoch": 0.05, "learning_rate": 0.0007671654964894685, "loss": 1.7346, "theoretical_loss": 4.484179005662186, "tokens_seen": 167116800 }, { "epoch": 0.05, "learning_rate": 0.0007671494483450351, "loss": 1.651, "theoretical_loss": 4.483954313490354, "tokens_seen": 167182336 }, { "epoch": 0.05, "learning_rate": 0.0007671334002006019, "loss": 1.7858, "theoretical_loss": 4.483729734032694, "tokens_seen": 167247872 }, { "epoch": 0.05, "learning_rate": 0.0007671173520561685, "loss": 1.7515, "theoretical_loss": 4.483505267188528, "tokens_seen": 167313408 }, { "epoch": 0.05, "learning_rate": 0.0007671013039117353, "loss": 1.7517, "theoretical_loss": 4.48328091285731, "tokens_seen": 167378944 }, { "epoch": 0.05, "learning_rate": 0.0007670852557673019, "loss": 1.8486, "theoretical_loss": 4.483056670938625, "tokens_seen": 167444480 }, { "epoch": 0.05, "learning_rate": 0.0007670692076228687, "loss": 1.796, "theoretical_loss": 4.482832541332183, "tokens_seen": 167510016 }, { "epoch": 0.05, "learning_rate": 0.0007670531594784353, "loss": 1.7272, "theoretical_loss": 4.482608523937824, "tokens_seen": 167575552 }, { "epoch": 0.05, "learning_rate": 0.0007670371113340021, "loss": 1.6783, "theoretical_loss": 4.4823846186555185, "tokens_seen": 167641088 }, { "epoch": 0.05, "learning_rate": 0.0007670210631895687, "loss": 1.658, "theoretical_loss": 4.482160825385363, "tokens_seen": 167706624 }, { "epoch": 0.05, "learning_rate": 0.0007670050150451355, "loss": 1.6597, "theoretical_loss": 4.481937144027583, "tokens_seen": 167772160 }, { "epoch": 0.05, "learning_rate": 0.0007669889669007021, "loss": 1.7993, "theoretical_loss": 4.48171357448253, "tokens_seen": 167837696 }, { "epoch": 0.05, "learning_rate": 0.0007669729187562689, "loss": 1.5537, "theoretical_loss": 4.481490116650686, "tokens_seen": 167903232 }, { "epoch": 0.05, "learning_rate": 0.0007669568706118355, "loss": 1.6828, "theoretical_loss": 4.48126677043266, "tokens_seen": 167968768 }, { "epoch": 0.05, "learning_rate": 0.0007669408224674023, "loss": 1.7082, "theoretical_loss": 4.481043535729183, "tokens_seen": 168034304 }, { "epoch": 0.05, "learning_rate": 0.0007669247743229689, "loss": 1.75, "theoretical_loss": 4.480820412441122, "tokens_seen": 168099840 }, { "epoch": 0.05, "learning_rate": 0.0007669087261785357, "loss": 1.8383, "theoretical_loss": 4.4805974004694615, "tokens_seen": 168165376 }, { "epoch": 0.05, "learning_rate": 0.0007668926780341023, "loss": 1.7588, "theoretical_loss": 4.48037449971532, "tokens_seen": 168230912 }, { "epoch": 0.05, "learning_rate": 0.0007668766298896691, "loss": 1.7177, "theoretical_loss": 4.480151710079937, "tokens_seen": 168296448 }, { "epoch": 0.05, "learning_rate": 0.0007668605817452357, "loss": 1.722, "theoretical_loss": 4.479929031464682, "tokens_seen": 168361984 }, { "epoch": 0.05, "learning_rate": 0.0007668445336008025, "loss": 1.7248, "theoretical_loss": 4.479706463771047, "tokens_seen": 168427520 }, { "epoch": 0.05, "learning_rate": 0.0007668284854563691, "loss": 1.6552, "theoretical_loss": 4.479484006900652, "tokens_seen": 168493056 }, { "epoch": 0.05, "learning_rate": 0.0007668124373119359, "loss": 1.6428, "theoretical_loss": 4.479261660755242, "tokens_seen": 168558592 }, { "epoch": 0.05, "learning_rate": 0.0007667963891675025, "loss": 1.8342, "theoretical_loss": 4.479039425236687, "tokens_seen": 168624128 }, { "epoch": 0.05, "learning_rate": 0.0007667803410230693, "loss": 1.7043, "theoretical_loss": 4.478817300246982, "tokens_seen": 168689664 }, { "epoch": 0.05, "learning_rate": 0.0007667642928786359, "loss": 1.7463, "theoretical_loss": 4.478595285688247, "tokens_seen": 168755200 }, { "epoch": 0.05, "learning_rate": 0.0007667482447342027, "loss": 1.6112, "theoretical_loss": 4.478373381462728, "tokens_seen": 168820736 }, { "epoch": 0.05, "learning_rate": 0.0007667321965897693, "loss": 1.689, "theoretical_loss": 4.478151587472791, "tokens_seen": 168886272 }, { "epoch": 0.05, "learning_rate": 0.0007667161484453361, "loss": 1.7633, "theoretical_loss": 4.477929903620931, "tokens_seen": 168951808 }, { "epoch": 0.05, "learning_rate": 0.0007667001003009027, "loss": 1.7455, "theoretical_loss": 4.477708329809766, "tokens_seen": 169017344 }, { "epoch": 0.05, "learning_rate": 0.0007666840521564695, "loss": 1.7587, "theoretical_loss": 4.477486865942036, "tokens_seen": 169082880 }, { "epoch": 0.05, "learning_rate": 0.0007666680040120361, "loss": 1.8086, "theoretical_loss": 4.477265511920607, "tokens_seen": 169148416 }, { "epoch": 0.05, "learning_rate": 0.0007666519558676029, "loss": 1.7175, "theoretical_loss": 4.477044267648466, "tokens_seen": 169213952 }, { "epoch": 0.05, "learning_rate": 0.0007666359077231695, "loss": 1.8273, "theoretical_loss": 4.476823133028725, "tokens_seen": 169279488 }, { "epoch": 0.05, "learning_rate": 0.0007666198595787363, "loss": 1.7291, "theoretical_loss": 4.47660210796462, "tokens_seen": 169345024 }, { "epoch": 0.05, "learning_rate": 0.0007666038114343029, "loss": 1.7855, "theoretical_loss": 4.476381192359504, "tokens_seen": 169410560 }, { "epoch": 0.05, "learning_rate": 0.0007665877632898697, "loss": 1.8167, "theoretical_loss": 4.476160386116861, "tokens_seen": 169476096 }, { "epoch": 0.05, "learning_rate": 0.0007665717151454363, "loss": 1.8135, "theoretical_loss": 4.475939689140291, "tokens_seen": 169541632 }, { "epoch": 0.05, "learning_rate": 0.0007665556670010031, "loss": 1.6252, "theoretical_loss": 4.475719101333519, "tokens_seen": 169607168 }, { "epoch": 0.05, "learning_rate": 0.0007665396188565697, "loss": 1.851, "theoretical_loss": 4.475498622600391, "tokens_seen": 169672704 }, { "epoch": 0.05, "learning_rate": 0.0007665235707121365, "loss": 1.642, "theoretical_loss": 4.475278252844875, "tokens_seen": 169738240 }, { "epoch": 0.05, "learning_rate": 0.0007665075225677031, "loss": 1.7247, "theoretical_loss": 4.475057991971062, "tokens_seen": 169803776 }, { "epoch": 0.05, "learning_rate": 0.0007664914744232699, "loss": 1.729, "theoretical_loss": 4.474837839883162, "tokens_seen": 169869312 }, { "epoch": 0.05, "learning_rate": 0.0007664754262788365, "loss": 1.7534, "theoretical_loss": 4.474617796485507, "tokens_seen": 169934848 }, { "epoch": 0.05, "learning_rate": 0.0007664593781344033, "loss": 1.7936, "theoretical_loss": 4.474397861682552, "tokens_seen": 170000384 }, { "epoch": 0.05, "learning_rate": 0.0007664433299899699, "loss": 1.747, "theoretical_loss": 4.474178035378869, "tokens_seen": 170065920 }, { "epoch": 0.05, "learning_rate": 0.0007664272818455367, "loss": 1.8412, "theoretical_loss": 4.473958317479154, "tokens_seen": 170131456 }, { "epoch": 0.05, "learning_rate": 0.0007664112337011033, "loss": 1.5907, "theoretical_loss": 4.473738707888221, "tokens_seen": 170196992 }, { "epoch": 0.05, "learning_rate": 0.0007663951855566701, "loss": 1.778, "theoretical_loss": 4.473519206511006, "tokens_seen": 170262528 }, { "epoch": 0.05, "learning_rate": 0.0007663791374122367, "loss": 1.8488, "theoretical_loss": 4.473299813252565, "tokens_seen": 170328064 }, { "epoch": 0.05, "objective/train/docs_used": 63171, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6936179399490356, "objective/train/theoretical_loss": 4.473080528018071, "objective/train/tokens_used": 190853600, "theoretical_loss": 4.473080528018071, "tokens_seen": 170393600 }, { "epoch": 0.05, "learning_rate": 0.0007663630892678035, "loss": 1.7195, "theoretical_loss": 4.473080528018071, "tokens_seen": 170393600 }, { "epoch": 0.05, "learning_rate": 0.0007663470411233702, "loss": 1.7942, "theoretical_loss": 4.4728613507128205, "tokens_seen": 170459136 }, { "epoch": 0.05, "learning_rate": 0.0007663309929789369, "loss": 1.673, "theoretical_loss": 4.472642281242226, "tokens_seen": 170524672 }, { "epoch": 0.05, "learning_rate": 0.0007663149448345036, "loss": 1.6945, "theoretical_loss": 4.472423319511822, "tokens_seen": 170590208 }, { "epoch": 0.05, "learning_rate": 0.0007662988966900703, "loss": 1.7537, "theoretical_loss": 4.472204465427261, "tokens_seen": 170655744 }, { "epoch": 0.05, "learning_rate": 0.000766282848545637, "loss": 1.7052, "theoretical_loss": 4.471985718894312, "tokens_seen": 170721280 }, { "epoch": 0.05, "learning_rate": 0.0007662668004012037, "loss": 1.7989, "theoretical_loss": 4.471767079818866, "tokens_seen": 170786816 }, { "epoch": 0.05, "learning_rate": 0.0007662507522567703, "loss": 1.7626, "theoretical_loss": 4.471548548106932, "tokens_seen": 170852352 }, { "epoch": 0.05, "learning_rate": 0.0007662347041123371, "loss": 1.7287, "theoretical_loss": 4.471330123664634, "tokens_seen": 170917888 }, { "epoch": 0.05, "learning_rate": 0.0007662186559679037, "loss": 1.5974, "theoretical_loss": 4.471111806398218, "tokens_seen": 170983424 }, { "epoch": 0.05, "learning_rate": 0.0007662026078234705, "loss": 1.7094, "theoretical_loss": 4.4708935962140455, "tokens_seen": 171048960 }, { "epoch": 0.05, "learning_rate": 0.0007661865596790371, "loss": 1.7946, "theoretical_loss": 4.4706754930185975, "tokens_seen": 171114496 }, { "epoch": 0.05, "learning_rate": 0.0007661705115346039, "loss": 1.677, "theoretical_loss": 4.47045749671847, "tokens_seen": 171180032 }, { "epoch": 0.05, "learning_rate": 0.0007661544633901705, "loss": 1.6607, "theoretical_loss": 4.470239607220377, "tokens_seen": 171245568 }, { "epoch": 0.05, "learning_rate": 0.0007661384152457373, "loss": 1.7783, "theoretical_loss": 4.470021824431152, "tokens_seen": 171311104 }, { "epoch": 0.05, "learning_rate": 0.0007661223671013039, "loss": 1.7274, "theoretical_loss": 4.469804148257742, "tokens_seen": 171376640 }, { "epoch": 0.05, "learning_rate": 0.0007661063189568706, "loss": 1.8638, "theoretical_loss": 4.469586578607213, "tokens_seen": 171442176 }, { "epoch": 0.05, "learning_rate": 0.0007660902708124373, "loss": 1.7533, "theoretical_loss": 4.469369115386746, "tokens_seen": 171507712 }, { "epoch": 0.05, "learning_rate": 0.000766074222668004, "loss": 1.7417, "theoretical_loss": 4.46915175850364, "tokens_seen": 171573248 }, { "epoch": 0.05, "learning_rate": 0.0007660581745235707, "loss": 1.6826, "theoretical_loss": 4.468934507865308, "tokens_seen": 171638784 }, { "epoch": 0.05, "learning_rate": 0.0007660421263791374, "loss": 1.7231, "theoretical_loss": 4.46871736337928, "tokens_seen": 171704320 }, { "epoch": 0.05, "learning_rate": 0.0007660260782347041, "loss": 1.7589, "theoretical_loss": 4.468500324953202, "tokens_seen": 171769856 }, { "epoch": 0.05, "learning_rate": 0.0007660100300902708, "loss": 1.8372, "theoretical_loss": 4.468283392494836, "tokens_seen": 171835392 }, { "epoch": 0.05, "learning_rate": 0.0007659939819458375, "loss": 1.8463, "theoretical_loss": 4.468066565912057, "tokens_seen": 171900928 }, { "epoch": 0.05, "learning_rate": 0.0007659779338014042, "loss": 1.7143, "theoretical_loss": 4.467849845112859, "tokens_seen": 171966464 }, { "epoch": 0.05, "learning_rate": 0.000765961885656971, "loss": 1.7111, "theoretical_loss": 4.467633230005347, "tokens_seen": 172032000 }, { "epoch": 0.05, "learning_rate": 0.0007659458375125376, "loss": 1.7905, "theoretical_loss": 4.467416720497743, "tokens_seen": 172097536 }, { "epoch": 0.05, "learning_rate": 0.0007659297893681043, "loss": 1.7305, "theoretical_loss": 4.467200316498385, "tokens_seen": 172163072 }, { "epoch": 0.05, "learning_rate": 0.000765913741223671, "loss": 1.6837, "theoretical_loss": 4.4669840179157205, "tokens_seen": 172228608 }, { "epoch": 0.05, "learning_rate": 0.0007658976930792377, "loss": 1.6952, "theoretical_loss": 4.466767824658317, "tokens_seen": 172294144 }, { "epoch": 0.05, "learning_rate": 0.0007658816449348044, "loss": 1.6608, "theoretical_loss": 4.466551736634852, "tokens_seen": 172359680 }, { "epoch": 0.05, "learning_rate": 0.0007658655967903711, "loss": 1.7433, "theoretical_loss": 4.466335753754118, "tokens_seen": 172425216 }, { "epoch": 0.05, "learning_rate": 0.0007658495486459378, "loss": 1.7105, "theoretical_loss": 4.466119875925022, "tokens_seen": 172490752 }, { "epoch": 0.05, "learning_rate": 0.0007658335005015046, "loss": 1.6972, "theoretical_loss": 4.465904103056586, "tokens_seen": 172556288 }, { "epoch": 0.05, "learning_rate": 0.0007658174523570712, "loss": 1.7239, "theoretical_loss": 4.465688435057938, "tokens_seen": 172621824 }, { "epoch": 0.05, "learning_rate": 0.000765801404212638, "loss": 1.6552, "theoretical_loss": 4.46547287183833, "tokens_seen": 172687360 }, { "epoch": 0.05, "learning_rate": 0.0007657853560682046, "loss": 1.5746, "theoretical_loss": 4.465257413307119, "tokens_seen": 172752896 }, { "epoch": 0.05, "learning_rate": 0.0007657693079237714, "loss": 1.7696, "theoretical_loss": 4.465042059373776, "tokens_seen": 172818432 }, { "epoch": 0.05, "learning_rate": 0.000765753259779338, "loss": 1.7059, "theoretical_loss": 4.464826809947887, "tokens_seen": 172883968 }, { "epoch": 0.05, "learning_rate": 0.0007657372116349048, "loss": 1.7048, "theoretical_loss": 4.464611664939149, "tokens_seen": 172949504 }, { "epoch": 0.05, "learning_rate": 0.0007657211634904714, "loss": 1.6989, "theoretical_loss": 4.46439662425737, "tokens_seen": 173015040 }, { "epoch": 0.05, "learning_rate": 0.0007657051153460382, "loss": 1.6436, "theoretical_loss": 4.464181687812474, "tokens_seen": 173080576 }, { "epoch": 0.05, "learning_rate": 0.0007656890672016048, "loss": 1.7972, "theoretical_loss": 4.4639668555144905, "tokens_seen": 173146112 }, { "epoch": 0.05, "learning_rate": 0.0007656730190571716, "loss": 1.7853, "theoretical_loss": 4.463752127273567, "tokens_seen": 173211648 }, { "epoch": 0.05, "learning_rate": 0.0007656569709127382, "loss": 1.795, "theoretical_loss": 4.463537502999959, "tokens_seen": 173277184 }, { "epoch": 0.05, "learning_rate": 0.000765640922768305, "loss": 1.6484, "theoretical_loss": 4.4633229826040335, "tokens_seen": 173342720 }, { "epoch": 0.05, "learning_rate": 0.0007656248746238716, "loss": 1.6078, "theoretical_loss": 4.46310856599627, "tokens_seen": 173408256 }, { "epoch": 0.05, "learning_rate": 0.0007656088264794384, "loss": 1.7552, "theoretical_loss": 4.462894253087258, "tokens_seen": 173473792 }, { "epoch": 0.05, "learning_rate": 0.000765592778335005, "loss": 1.8263, "theoretical_loss": 4.462680043787699, "tokens_seen": 173539328 }, { "epoch": 0.05, "learning_rate": 0.0007655767301905718, "loss": 1.794, "theoretical_loss": 4.4624659380084015, "tokens_seen": 173604864 }, { "epoch": 0.05, "objective/train/docs_used": 64355, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7965178489685059, "objective/train/theoretical_loss": 4.462251935660287, "objective/train/tokens_used": 194130400, "theoretical_loss": 4.462251935660287, "tokens_seen": 173670400 }, { "epoch": 0.05, "learning_rate": 0.0007655606820461384, "loss": 1.6888, "theoretical_loss": 4.462251935660287, "tokens_seen": 173670400 }, { "epoch": 0.05, "learning_rate": 0.0007655446339017052, "loss": 1.722, "theoretical_loss": 4.46203803665439, "tokens_seen": 173735936 }, { "epoch": 0.05, "learning_rate": 0.0007655285857572718, "loss": 1.8336, "theoretical_loss": 4.4618242409018505, "tokens_seen": 173801472 }, { "epoch": 0.05, "learning_rate": 0.0007655125376128386, "loss": 1.7571, "theoretical_loss": 4.461610548313919, "tokens_seen": 173867008 }, { "epoch": 0.05, "learning_rate": 0.0007654964894684052, "loss": 1.7694, "theoretical_loss": 4.461396958801959, "tokens_seen": 173932544 }, { "epoch": 0.05, "learning_rate": 0.000765480441323972, "loss": 1.7376, "theoretical_loss": 4.461183472277439, "tokens_seen": 173998080 }, { "epoch": 0.05, "learning_rate": 0.0007654643931795386, "loss": 1.6581, "theoretical_loss": 4.460970088651941, "tokens_seen": 174063616 }, { "epoch": 0.05, "learning_rate": 0.0007654483450351054, "loss": 1.6625, "theoretical_loss": 4.460756807837154, "tokens_seen": 174129152 }, { "epoch": 0.05, "learning_rate": 0.000765432296890672, "loss": 1.6933, "theoretical_loss": 4.460543629744874, "tokens_seen": 174194688 }, { "epoch": 0.05, "learning_rate": 0.0007654162487462388, "loss": 1.4361, "theoretical_loss": 4.460330554287011, "tokens_seen": 174260224 }, { "epoch": 0.05, "learning_rate": 0.0007654002006018054, "loss": 1.7742, "theoretical_loss": 4.460117581375579, "tokens_seen": 174325760 }, { "epoch": 0.05, "learning_rate": 0.0007653841524573722, "loss": 1.6683, "theoretical_loss": 4.459904710922704, "tokens_seen": 174391296 }, { "epoch": 0.05, "learning_rate": 0.0007653681043129388, "loss": 1.7873, "theoretical_loss": 4.459691942840616, "tokens_seen": 174456832 }, { "epoch": 0.05, "learning_rate": 0.0007653520561685056, "loss": 1.7836, "theoretical_loss": 4.459479277041657, "tokens_seen": 174522368 }, { "epoch": 0.05, "learning_rate": 0.0007653360080240722, "loss": 1.616, "theoretical_loss": 4.459266713438277, "tokens_seen": 174587904 }, { "epoch": 0.05, "learning_rate": 0.000765319959879639, "loss": 1.7366, "theoretical_loss": 4.459054251943029, "tokens_seen": 174653440 }, { "epoch": 0.05, "learning_rate": 0.0007653039117352056, "loss": 1.7021, "theoretical_loss": 4.4588418924685795, "tokens_seen": 174718976 }, { "epoch": 0.05, "learning_rate": 0.0007652878635907724, "loss": 1.9371, "theoretical_loss": 4.458629634927698, "tokens_seen": 174784512 }, { "epoch": 0.05, "learning_rate": 0.000765271815446339, "loss": 1.6323, "theoretical_loss": 4.458417479233265, "tokens_seen": 174850048 }, { "epoch": 0.05, "learning_rate": 0.0007652557673019058, "loss": 1.7393, "theoretical_loss": 4.4582054252982655, "tokens_seen": 174915584 }, { "epoch": 0.05, "learning_rate": 0.0007652397191574724, "loss": 1.7396, "theoretical_loss": 4.457993473035792, "tokens_seen": 174981120 }, { "epoch": 0.05, "learning_rate": 0.0007652236710130392, "loss": 1.6274, "theoretical_loss": 4.457781622359043, "tokens_seen": 175046656 }, { "epoch": 0.05, "learning_rate": 0.0007652076228686058, "loss": 1.6795, "theoretical_loss": 4.457569873181326, "tokens_seen": 175112192 }, { "epoch": 0.05, "learning_rate": 0.0007651915747241726, "loss": 1.7442, "theoretical_loss": 4.457358225416052, "tokens_seen": 175177728 }, { "epoch": 0.05, "learning_rate": 0.0007651755265797392, "loss": 1.6656, "theoretical_loss": 4.4571466789767396, "tokens_seen": 175243264 }, { "epoch": 0.05, "learning_rate": 0.000765159478435306, "loss": 1.6225, "theoretical_loss": 4.456935233777013, "tokens_seen": 175308800 }, { "epoch": 0.05, "learning_rate": 0.0007651434302908727, "loss": 1.7749, "theoretical_loss": 4.456723889730605, "tokens_seen": 175374336 }, { "epoch": 0.05, "learning_rate": 0.0007651273821464394, "loss": 1.7267, "theoretical_loss": 4.456512646751349, "tokens_seen": 175439872 }, { "epoch": 0.05, "learning_rate": 0.0007651113340020061, "loss": 1.7734, "theoretical_loss": 4.456301504753188, "tokens_seen": 175505408 }, { "epoch": 0.05, "learning_rate": 0.0007650952858575728, "loss": 1.7111, "theoretical_loss": 4.456090463650169, "tokens_seen": 175570944 }, { "epoch": 0.05, "learning_rate": 0.0007650792377131395, "loss": 1.6561, "theoretical_loss": 4.455879523356444, "tokens_seen": 175636480 }, { "epoch": 0.05, "learning_rate": 0.0007650631895687062, "loss": 1.77, "theoretical_loss": 4.4556686837862705, "tokens_seen": 175702016 }, { "epoch": 0.05, "learning_rate": 0.0007650471414242729, "loss": 1.6699, "theoretical_loss": 4.455457944854011, "tokens_seen": 175767552 }, { "epoch": 0.05, "learning_rate": 0.0007650310932798396, "loss": 1.7107, "theoretical_loss": 4.455247306474131, "tokens_seen": 175833088 }, { "epoch": 0.05, "learning_rate": 0.0007650150451354063, "loss": 1.8197, "theoretical_loss": 4.455036768561204, "tokens_seen": 175898624 }, { "epoch": 0.05, "learning_rate": 0.000764998996990973, "loss": 1.7647, "theoretical_loss": 4.4548263310299046, "tokens_seen": 175964160 }, { "epoch": 0.05, "learning_rate": 0.0007649829488465397, "loss": 1.7439, "theoretical_loss": 4.454615993795015, "tokens_seen": 176029696 }, { "epoch": 0.05, "learning_rate": 0.0007649669007021064, "loss": 1.6774, "theoretical_loss": 4.454405756771415, "tokens_seen": 176095232 }, { "epoch": 0.05, "learning_rate": 0.0007649508525576731, "loss": 1.822, "theoretical_loss": 4.454195619874096, "tokens_seen": 176160768 }, { "epoch": 0.05, "learning_rate": 0.0007649348044132398, "loss": 1.7231, "theoretical_loss": 4.453985583018149, "tokens_seen": 176226304 }, { "epoch": 0.05, "learning_rate": 0.0007649187562688065, "loss": 1.7635, "theoretical_loss": 4.4537756461187685, "tokens_seen": 176291840 }, { "epoch": 0.05, "learning_rate": 0.0007649027081243732, "loss": 1.7327, "theoretical_loss": 4.453565809091254, "tokens_seen": 176357376 }, { "epoch": 0.05, "learning_rate": 0.0007648866599799399, "loss": 1.7449, "theoretical_loss": 4.453356071851006, "tokens_seen": 176422912 }, { "epoch": 0.05, "learning_rate": 0.0007648706118355066, "loss": 1.7211, "theoretical_loss": 4.453146434313531, "tokens_seen": 176488448 }, { "epoch": 0.05, "learning_rate": 0.0007648545636910732, "loss": 1.851, "theoretical_loss": 4.452936896394435, "tokens_seen": 176553984 }, { "epoch": 0.05, "learning_rate": 0.00076483851554664, "loss": 1.7872, "theoretical_loss": 4.452727458009431, "tokens_seen": 176619520 }, { "epoch": 0.05, "learning_rate": 0.0007648224674022066, "loss": 1.7427, "theoretical_loss": 4.45251811907433, "tokens_seen": 176685056 }, { "epoch": 0.05, "learning_rate": 0.0007648064192577734, "loss": 1.6532, "theoretical_loss": 4.452308879505047, "tokens_seen": 176750592 }, { "epoch": 0.05, "learning_rate": 0.00076479037111334, "loss": 1.8032, "theoretical_loss": 4.452099739217601, "tokens_seen": 176816128 }, { "epoch": 0.05, "learning_rate": 0.0007647743229689068, "loss": 1.6829, "theoretical_loss": 4.451890698128112, "tokens_seen": 176881664 }, { "epoch": 0.05, "objective/train/docs_used": 65665, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5994759798049927, "objective/train/theoretical_loss": 4.4516817561528015, "objective/train/tokens_used": 197407200, "theoretical_loss": 4.4516817561528015, "tokens_seen": 176947200 }, { "epoch": 0.05, "learning_rate": 0.0007647582748244734, "loss": 1.6761, "theoretical_loss": 4.4516817561528015, "tokens_seen": 176947200 }, { "epoch": 0.05, "learning_rate": 0.0007647422266800402, "loss": 1.6801, "theoretical_loss": 4.451472913207992, "tokens_seen": 177012736 }, { "epoch": 0.05, "learning_rate": 0.0007647261785356068, "loss": 1.7034, "theoretical_loss": 4.451264169210107, "tokens_seen": 177078272 }, { "epoch": 0.05, "learning_rate": 0.0007647101303911735, "loss": 1.651, "theoretical_loss": 4.451055524075677, "tokens_seen": 177143808 }, { "epoch": 0.05, "learning_rate": 0.0007646940822467402, "loss": 1.6878, "theoretical_loss": 4.450846977721326, "tokens_seen": 177209344 }, { "epoch": 0.05, "learning_rate": 0.0007646780341023069, "loss": 1.7719, "theoretical_loss": 4.450638530063786, "tokens_seen": 177274880 }, { "epoch": 0.05, "learning_rate": 0.0007646619859578736, "loss": 1.7237, "theoretical_loss": 4.450430181019884, "tokens_seen": 177340416 }, { "epoch": 0.05, "learning_rate": 0.0007646459378134403, "loss": 1.7421, "theoretical_loss": 4.450221930506553, "tokens_seen": 177405952 }, { "epoch": 0.05, "learning_rate": 0.000764629889669007, "loss": 1.6159, "theoretical_loss": 4.450013778440822, "tokens_seen": 177471488 }, { "epoch": 0.05, "learning_rate": 0.0007646138415245737, "loss": 1.6349, "theoretical_loss": 4.449805724739824, "tokens_seen": 177537024 }, { "epoch": 0.05, "learning_rate": 0.0007645977933801404, "loss": 1.6987, "theoretical_loss": 4.449597769320791, "tokens_seen": 177602560 }, { "epoch": 0.05, "learning_rate": 0.0007645817452357071, "loss": 1.7311, "theoretical_loss": 4.4493899121010525, "tokens_seen": 177668096 }, { "epoch": 0.05, "learning_rate": 0.0007645656970912738, "loss": 1.7738, "theoretical_loss": 4.449182152998044, "tokens_seen": 177733632 }, { "epoch": 0.05, "learning_rate": 0.0007645496489468405, "loss": 1.6177, "theoretical_loss": 4.448974491929297, "tokens_seen": 177799168 }, { "epoch": 0.05, "learning_rate": 0.0007645336008024072, "loss": 1.8036, "theoretical_loss": 4.448766928812441, "tokens_seen": 177864704 }, { "epoch": 0.05, "learning_rate": 0.0007645175526579739, "loss": 1.7037, "theoretical_loss": 4.448559463565209, "tokens_seen": 177930240 }, { "epoch": 0.05, "learning_rate": 0.0007645015045135406, "loss": 1.6341, "theoretical_loss": 4.44835209610543, "tokens_seen": 177995776 }, { "epoch": 0.05, "learning_rate": 0.0007644854563691073, "loss": 1.7024, "theoretical_loss": 4.448144826351036, "tokens_seen": 178061312 }, { "epoch": 0.05, "learning_rate": 0.000764469408224674, "loss": 1.6771, "theoretical_loss": 4.447937654220054, "tokens_seen": 178126848 }, { "epoch": 0.05, "learning_rate": 0.0007644533600802407, "loss": 1.7449, "theoretical_loss": 4.447730579630612, "tokens_seen": 178192384 }, { "epoch": 0.05, "learning_rate": 0.0007644373119358075, "loss": 1.5651, "theoretical_loss": 4.447523602500936, "tokens_seen": 178257920 }, { "epoch": 0.05, "learning_rate": 0.0007644212637913741, "loss": 1.6465, "theoretical_loss": 4.447316722749351, "tokens_seen": 178323456 }, { "epoch": 0.05, "learning_rate": 0.0007644052156469409, "loss": 1.6778, "theoretical_loss": 4.447109940294283, "tokens_seen": 178388992 }, { "epoch": 0.05, "learning_rate": 0.0007643891675025075, "loss": 1.7214, "theoretical_loss": 4.44690325505425, "tokens_seen": 178454528 }, { "epoch": 0.05, "learning_rate": 0.0007643731193580743, "loss": 1.6484, "theoretical_loss": 4.446696666947874, "tokens_seen": 178520064 }, { "epoch": 0.05, "learning_rate": 0.0007643570712136409, "loss": 1.733, "theoretical_loss": 4.446490175893872, "tokens_seen": 178585600 }, { "epoch": 0.05, "learning_rate": 0.0007643410230692077, "loss": 1.6846, "theoretical_loss": 4.44628378181106, "tokens_seen": 178651136 }, { "epoch": 0.05, "learning_rate": 0.0007643249749247743, "loss": 1.7986, "theoretical_loss": 4.4460774846183515, "tokens_seen": 178716672 }, { "epoch": 0.05, "learning_rate": 0.0007643089267803411, "loss": 1.6765, "theoretical_loss": 4.445871284234757, "tokens_seen": 178782208 }, { "epoch": 0.05, "learning_rate": 0.0007642928786359077, "loss": 1.8968, "theoretical_loss": 4.445665180579383, "tokens_seen": 178847744 }, { "epoch": 0.05, "learning_rate": 0.0007642768304914745, "loss": 1.6697, "theoretical_loss": 4.445459173571438, "tokens_seen": 178913280 }, { "epoch": 0.05, "learning_rate": 0.0007642607823470411, "loss": 1.731, "theoretical_loss": 4.445253263130223, "tokens_seen": 178978816 }, { "epoch": 0.05, "learning_rate": 0.0007642447342026079, "loss": 1.668, "theoretical_loss": 4.445047449175136, "tokens_seen": 179044352 }, { "epoch": 0.05, "learning_rate": 0.0007642286860581745, "loss": 1.6233, "theoretical_loss": 4.444841731625674, "tokens_seen": 179109888 }, { "epoch": 0.05, "learning_rate": 0.0007642126379137413, "loss": 1.8112, "theoretical_loss": 4.444636110401431, "tokens_seen": 179175424 }, { "epoch": 0.05, "learning_rate": 0.0007641965897693079, "loss": 1.7487, "theoretical_loss": 4.444430585422094, "tokens_seen": 179240960 }, { "epoch": 0.05, "learning_rate": 0.0007641805416248747, "loss": 1.5881, "theoretical_loss": 4.444225156607449, "tokens_seen": 179306496 }, { "epoch": 0.05, "learning_rate": 0.0007641644934804413, "loss": 1.7401, "theoretical_loss": 4.444019823877378, "tokens_seen": 179372032 }, { "epoch": 0.05, "learning_rate": 0.0007641484453360081, "loss": 1.6739, "theoretical_loss": 4.4438145871518575, "tokens_seen": 179437568 }, { "epoch": 0.05, "learning_rate": 0.0007641323971915747, "loss": 1.7123, "theoretical_loss": 4.443609446350962, "tokens_seen": 179503104 }, { "epoch": 0.05, "learning_rate": 0.0007641163490471415, "loss": 1.697, "theoretical_loss": 4.44340440139486, "tokens_seen": 179568640 }, { "epoch": 0.05, "learning_rate": 0.0007641003009027081, "loss": 1.7086, "theoretical_loss": 4.443199452203816, "tokens_seen": 179634176 }, { "epoch": 0.05, "learning_rate": 0.0007640842527582749, "loss": 1.6203, "theoretical_loss": 4.442994598698189, "tokens_seen": 179699712 }, { "epoch": 0.05, "learning_rate": 0.0007640682046138415, "loss": 1.7007, "theoretical_loss": 4.4427898407984365, "tokens_seen": 179765248 }, { "epoch": 0.05, "learning_rate": 0.0007640521564694083, "loss": 1.6982, "theoretical_loss": 4.442585178425107, "tokens_seen": 179830784 }, { "epoch": 0.05, "learning_rate": 0.0007640361083249749, "loss": 1.6712, "theoretical_loss": 4.442380611498846, "tokens_seen": 179896320 }, { "epoch": 0.05, "learning_rate": 0.0007640200601805417, "loss": 1.7155, "theoretical_loss": 4.442176139940393, "tokens_seen": 179961856 }, { "epoch": 0.05, "learning_rate": 0.0007640040120361083, "loss": 1.8174, "theoretical_loss": 4.441971763670585, "tokens_seen": 180027392 }, { "epoch": 0.05, "learning_rate": 0.0007639879638916751, "loss": 1.8372, "theoretical_loss": 4.441767482610349, "tokens_seen": 180092928 }, { "epoch": 0.05, "learning_rate": 0.0007639719157472417, "loss": 1.7136, "theoretical_loss": 4.441563296680709, "tokens_seen": 180158464 }, { "epoch": 0.05, "objective/train/docs_used": 66635, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7017289400100708, "objective/train/theoretical_loss": 4.441359205802783, "objective/train/tokens_used": 200684000, "theoretical_loss": 4.441359205802783, "tokens_seen": 180224000 }, { "epoch": 0.05, "learning_rate": 0.0007639558676028085, "loss": 1.6671, "theoretical_loss": 4.441359205802783, "tokens_seen": 180224000 }, { "epoch": 0.05, "learning_rate": 0.0007639398194583752, "loss": 1.74, "theoretical_loss": 4.441155209897782, "tokens_seen": 180289536 }, { "epoch": 0.05, "learning_rate": 0.0007639237713139419, "loss": 1.7344, "theoretical_loss": 4.440951308887014, "tokens_seen": 180355072 }, { "epoch": 0.05, "learning_rate": 0.0007639077231695086, "loss": 1.7618, "theoretical_loss": 4.440747502691876, "tokens_seen": 180420608 }, { "epoch": 0.05, "learning_rate": 0.0007638916750250753, "loss": 1.7888, "theoretical_loss": 4.440543791233862, "tokens_seen": 180486144 }, { "epoch": 0.05, "learning_rate": 0.000763875626880642, "loss": 1.6206, "theoretical_loss": 4.440340174434558, "tokens_seen": 180551680 }, { "epoch": 0.05, "learning_rate": 0.0007638595787362087, "loss": 1.7611, "theoretical_loss": 4.440136652215646, "tokens_seen": 180617216 }, { "epoch": 0.05, "learning_rate": 0.0007638435305917754, "loss": 1.7102, "theoretical_loss": 4.439933224498898, "tokens_seen": 180682752 }, { "epoch": 0.05, "learning_rate": 0.0007638274824473421, "loss": 1.7108, "theoretical_loss": 4.43972989120618, "tokens_seen": 180748288 }, { "epoch": 0.05, "learning_rate": 0.0007638114343029088, "loss": 1.8015, "theoretical_loss": 4.439526652259451, "tokens_seen": 180813824 }, { "epoch": 0.05, "learning_rate": 0.0007637953861584755, "loss": 1.6381, "theoretical_loss": 4.439323507580761, "tokens_seen": 180879360 }, { "epoch": 0.05, "learning_rate": 0.0007637793380140422, "loss": 1.633, "theoretical_loss": 4.439120457092258, "tokens_seen": 180944896 }, { "epoch": 0.05, "learning_rate": 0.0007637632898696089, "loss": 1.6394, "theoretical_loss": 4.438917500716178, "tokens_seen": 181010432 }, { "epoch": 0.05, "learning_rate": 0.0007637472417251756, "loss": 1.658, "theoretical_loss": 4.438714638374849, "tokens_seen": 181075968 }, { "epoch": 0.05, "learning_rate": 0.0007637311935807423, "loss": 1.7647, "theoretical_loss": 4.438511869990693, "tokens_seen": 181141504 }, { "epoch": 0.05, "learning_rate": 0.000763715145436309, "loss": 1.7654, "theoretical_loss": 4.438309195486223, "tokens_seen": 181207040 }, { "epoch": 0.05, "learning_rate": 0.0007636990972918757, "loss": 1.7345, "theoretical_loss": 4.438106614784045, "tokens_seen": 181272576 }, { "epoch": 0.05, "learning_rate": 0.0007636830491474424, "loss": 1.7755, "theoretical_loss": 4.437904127806856, "tokens_seen": 181338112 }, { "epoch": 0.05, "learning_rate": 0.0007636670010030091, "loss": 1.7887, "theoretical_loss": 4.437701734477445, "tokens_seen": 181403648 }, { "epoch": 0.05, "learning_rate": 0.0007636509528585758, "loss": 1.7479, "theoretical_loss": 4.437499434718692, "tokens_seen": 181469184 }, { "epoch": 0.06, "learning_rate": 0.0007636349047141425, "loss": 1.8136, "theoretical_loss": 4.437297228453567, "tokens_seen": 181534720 }, { "epoch": 0.06, "learning_rate": 0.0007636188565697092, "loss": 1.6026, "theoretical_loss": 4.4370951156051355, "tokens_seen": 181600256 }, { "epoch": 0.06, "learning_rate": 0.0007636028084252759, "loss": 1.7126, "theoretical_loss": 4.43689309609655, "tokens_seen": 181665792 }, { "epoch": 0.06, "learning_rate": 0.0007635867602808426, "loss": 1.7019, "theoretical_loss": 4.436691169851055, "tokens_seen": 181731328 }, { "epoch": 0.06, "learning_rate": 0.0007635707121364093, "loss": 1.5814, "theoretical_loss": 4.436489336791985, "tokens_seen": 181796864 }, { "epoch": 0.06, "learning_rate": 0.000763554663991976, "loss": 1.7454, "theoretical_loss": 4.436287596842769, "tokens_seen": 181862400 }, { "epoch": 0.06, "learning_rate": 0.0007635386158475427, "loss": 1.6315, "theoretical_loss": 4.4360859499269205, "tokens_seen": 181927936 }, { "epoch": 0.06, "learning_rate": 0.0007635225677031094, "loss": 1.7988, "theoretical_loss": 4.435884395968047, "tokens_seen": 181993472 }, { "epoch": 0.06, "learning_rate": 0.0007635065195586761, "loss": 1.763, "theoretical_loss": 4.435682934889847, "tokens_seen": 182059008 }, { "epoch": 0.06, "learning_rate": 0.0007634904714142428, "loss": 1.7406, "theoretical_loss": 4.435481566616108, "tokens_seen": 182124544 }, { "epoch": 0.06, "learning_rate": 0.0007634744232698095, "loss": 1.605, "theoretical_loss": 4.435280291070705, "tokens_seen": 182190080 }, { "epoch": 0.06, "learning_rate": 0.0007634583751253762, "loss": 1.7259, "theoretical_loss": 4.435079108177606, "tokens_seen": 182255616 }, { "epoch": 0.06, "learning_rate": 0.0007634423269809429, "loss": 1.7148, "theoretical_loss": 4.434878017860869, "tokens_seen": 182321152 }, { "epoch": 0.06, "learning_rate": 0.0007634262788365095, "loss": 1.752, "theoretical_loss": 4.434677020044637, "tokens_seen": 182386688 }, { "epoch": 0.06, "learning_rate": 0.0007634102306920763, "loss": 1.7357, "theoretical_loss": 4.4344761146531475, "tokens_seen": 182452224 }, { "epoch": 0.06, "learning_rate": 0.0007633941825476429, "loss": 1.712, "theoretical_loss": 4.434275301610725, "tokens_seen": 182517760 }, { "epoch": 0.06, "learning_rate": 0.0007633781344032097, "loss": 1.7025, "theoretical_loss": 4.434074580841784, "tokens_seen": 182583296 }, { "epoch": 0.06, "learning_rate": 0.0007633620862587763, "loss": 1.6831, "theoretical_loss": 4.433873952270826, "tokens_seen": 182648832 }, { "epoch": 0.06, "learning_rate": 0.0007633460381143431, "loss": 1.6822, "theoretical_loss": 4.433673415822443, "tokens_seen": 182714368 }, { "epoch": 0.06, "learning_rate": 0.0007633299899699097, "loss": 1.6375, "theoretical_loss": 4.433472971421315, "tokens_seen": 182779904 }, { "epoch": 0.06, "learning_rate": 0.0007633139418254764, "loss": 1.7446, "theoretical_loss": 4.433272618992213, "tokens_seen": 182845440 }, { "epoch": 0.06, "learning_rate": 0.0007632978936810431, "loss": 1.6624, "theoretical_loss": 4.433072358459991, "tokens_seen": 182910976 }, { "epoch": 0.06, "learning_rate": 0.0007632818455366098, "loss": 1.667, "theoretical_loss": 4.432872189749596, "tokens_seen": 182976512 }, { "epoch": 0.06, "learning_rate": 0.0007632657973921765, "loss": 1.6365, "theoretical_loss": 4.432672112786063, "tokens_seen": 183042048 }, { "epoch": 0.06, "learning_rate": 0.0007632497492477432, "loss": 1.7046, "theoretical_loss": 4.4324721274945125, "tokens_seen": 183107584 }, { "epoch": 0.06, "learning_rate": 0.0007632337011033099, "loss": 1.6919, "theoretical_loss": 4.432272233800154, "tokens_seen": 183173120 }, { "epoch": 0.06, "learning_rate": 0.0007632176529588766, "loss": 1.6129, "theoretical_loss": 4.432072431628286, "tokens_seen": 183238656 }, { "epoch": 0.06, "learning_rate": 0.0007632016048144433, "loss": 1.6262, "theoretical_loss": 4.431872720904292, "tokens_seen": 183304192 }, { "epoch": 0.06, "learning_rate": 0.00076318555667001, "loss": 1.6003, "theoretical_loss": 4.4316731015536455, "tokens_seen": 183369728 }, { "epoch": 0.06, "learning_rate": 0.0007631695085255767, "loss": 1.6435, "theoretical_loss": 4.431473573501906, "tokens_seen": 183435264 }, { "epoch": 0.06, "objective/train/docs_used": 67570, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8887503147125244, "objective/train/theoretical_loss": 4.431274136674721, "objective/train/tokens_used": 203960800, "theoretical_loss": 4.431274136674721, "tokens_seen": 183500800 }, { "epoch": 0.06, "learning_rate": 0.0007631534603811434, "loss": 1.8266, "theoretical_loss": 4.431274136674721, "tokens_seen": 183500800 }, { "epoch": 0.06, "learning_rate": 0.0007631374122367101, "loss": 1.6985, "theoretical_loss": 4.431074790997823, "tokens_seen": 183566336 }, { "epoch": 0.06, "learning_rate": 0.0007631213640922768, "loss": 1.7259, "theoretical_loss": 4.430875536397036, "tokens_seen": 183631872 }, { "epoch": 0.06, "learning_rate": 0.0007631053159478436, "loss": 1.7227, "theoretical_loss": 4.430676372798265, "tokens_seen": 183697408 }, { "epoch": 0.06, "learning_rate": 0.0007630892678034102, "loss": 1.8484, "theoretical_loss": 4.430477300127507, "tokens_seen": 183762944 }, { "epoch": 0.06, "learning_rate": 0.000763073219658977, "loss": 1.7766, "theoretical_loss": 4.43027831831084, "tokens_seen": 183828480 }, { "epoch": 0.06, "learning_rate": 0.0007630571715145436, "loss": 1.7096, "theoretical_loss": 4.430079427274435, "tokens_seen": 183894016 }, { "epoch": 0.06, "learning_rate": 0.0007630411233701104, "loss": 1.6482, "theoretical_loss": 4.429880626944543, "tokens_seen": 183959552 }, { "epoch": 0.06, "learning_rate": 0.000763025075225677, "loss": 1.7298, "theoretical_loss": 4.429681917247506, "tokens_seen": 184025088 }, { "epoch": 0.06, "learning_rate": 0.0007630090270812438, "loss": 1.7268, "theoretical_loss": 4.429483298109749, "tokens_seen": 184090624 }, { "epoch": 0.06, "learning_rate": 0.0007629929789368104, "loss": 1.6922, "theoretical_loss": 4.429284769457784, "tokens_seen": 184156160 }, { "epoch": 0.06, "learning_rate": 0.0007629769307923772, "loss": 1.5606, "theoretical_loss": 4.429086331218208, "tokens_seen": 184221696 }, { "epoch": 0.06, "learning_rate": 0.0007629608826479438, "loss": 1.6067, "theoretical_loss": 4.428887983317706, "tokens_seen": 184287232 }, { "epoch": 0.06, "learning_rate": 0.0007629448345035106, "loss": 1.7639, "theoretical_loss": 4.4286897256830455, "tokens_seen": 184352768 }, { "epoch": 0.06, "learning_rate": 0.0007629287863590772, "loss": 1.7258, "theoretical_loss": 4.428491558241081, "tokens_seen": 184418304 }, { "epoch": 0.06, "learning_rate": 0.000762912738214644, "loss": 1.6622, "theoretical_loss": 4.428293480918751, "tokens_seen": 184483840 }, { "epoch": 0.06, "learning_rate": 0.0007628966900702106, "loss": 1.7358, "theoretical_loss": 4.428095493643081, "tokens_seen": 184549376 }, { "epoch": 0.06, "learning_rate": 0.0007628806419257774, "loss": 1.6846, "theoretical_loss": 4.427897596341181, "tokens_seen": 184614912 }, { "epoch": 0.06, "learning_rate": 0.000762864593781344, "loss": 1.6989, "theoretical_loss": 4.427699788940243, "tokens_seen": 184680448 }, { "epoch": 0.06, "learning_rate": 0.0007628485456369108, "loss": 1.6336, "theoretical_loss": 4.427502071367549, "tokens_seen": 184745984 }, { "epoch": 0.06, "learning_rate": 0.0007628324974924774, "loss": 1.684, "theoretical_loss": 4.42730444355046, "tokens_seen": 184811520 }, { "epoch": 0.06, "learning_rate": 0.0007628164493480442, "loss": 1.7098, "theoretical_loss": 4.4271069054164265, "tokens_seen": 184877056 }, { "epoch": 0.06, "learning_rate": 0.0007628004012036108, "loss": 1.7049, "theoretical_loss": 4.426909456892979, "tokens_seen": 184942592 }, { "epoch": 0.06, "learning_rate": 0.0007627843530591776, "loss": 1.6428, "theoretical_loss": 4.426712097907735, "tokens_seen": 185008128 }, { "epoch": 0.06, "learning_rate": 0.0007627683049147442, "loss": 1.6611, "theoretical_loss": 4.426514828388394, "tokens_seen": 185073664 }, { "epoch": 0.06, "learning_rate": 0.000762752256770311, "loss": 1.5817, "theoretical_loss": 4.426317648262743, "tokens_seen": 185139200 }, { "epoch": 0.06, "learning_rate": 0.0007627362086258777, "loss": 1.6968, "theoretical_loss": 4.426120557458649, "tokens_seen": 185204736 }, { "epoch": 0.06, "learning_rate": 0.0007627201604814444, "loss": 1.6527, "theoretical_loss": 4.425923555904065, "tokens_seen": 185270272 }, { "epoch": 0.06, "learning_rate": 0.0007627041123370111, "loss": 1.6712, "theoretical_loss": 4.4257266435270255, "tokens_seen": 185335808 }, { "epoch": 0.06, "learning_rate": 0.0007626880641925778, "loss": 1.622, "theoretical_loss": 4.425529820255651, "tokens_seen": 185401344 }, { "epoch": 0.06, "learning_rate": 0.0007626720160481445, "loss": 1.7908, "theoretical_loss": 4.425333086018144, "tokens_seen": 185466880 }, { "epoch": 0.06, "learning_rate": 0.0007626559679037112, "loss": 1.7091, "theoretical_loss": 4.425136440742788, "tokens_seen": 185532416 }, { "epoch": 0.06, "learning_rate": 0.0007626399197592779, "loss": 1.6781, "theoretical_loss": 4.424939884357956, "tokens_seen": 185597952 }, { "epoch": 0.06, "learning_rate": 0.0007626238716148446, "loss": 1.697, "theoretical_loss": 4.424743416792096, "tokens_seen": 185663488 }, { "epoch": 0.06, "learning_rate": 0.0007626078234704113, "loss": 1.6791, "theoretical_loss": 4.424547037973745, "tokens_seen": 185729024 }, { "epoch": 0.06, "learning_rate": 0.000762591775325978, "loss": 1.699, "theoretical_loss": 4.424350747831518, "tokens_seen": 185794560 }, { "epoch": 0.06, "learning_rate": 0.0007625757271815447, "loss": 1.7917, "theoretical_loss": 4.424154546294117, "tokens_seen": 185860096 }, { "epoch": 0.06, "learning_rate": 0.0007625596790371114, "loss": 1.707, "theoretical_loss": 4.4239584332903235, "tokens_seen": 185925632 }, { "epoch": 0.06, "learning_rate": 0.0007625436308926781, "loss": 1.6794, "theoretical_loss": 4.423762408749001, "tokens_seen": 185991168 }, { "epoch": 0.06, "learning_rate": 0.0007625275827482448, "loss": 1.6936, "theoretical_loss": 4.423566472599099, "tokens_seen": 186056704 }, { "epoch": 0.06, "learning_rate": 0.0007625115346038115, "loss": 1.8014, "theoretical_loss": 4.423370624769643, "tokens_seen": 186122240 }, { "epoch": 0.06, "learning_rate": 0.0007624954864593782, "loss": 1.7071, "theoretical_loss": 4.423174865189747, "tokens_seen": 186187776 }, { "epoch": 0.06, "learning_rate": 0.0007624794383149449, "loss": 1.6442, "theoretical_loss": 4.422979193788601, "tokens_seen": 186253312 }, { "epoch": 0.06, "learning_rate": 0.0007624633901705116, "loss": 1.5889, "theoretical_loss": 4.422783610495481, "tokens_seen": 186318848 }, { "epoch": 0.06, "learning_rate": 0.0007624473420260783, "loss": 1.6439, "theoretical_loss": 4.422588115239742, "tokens_seen": 186384384 }, { "epoch": 0.06, "learning_rate": 0.000762431293881645, "loss": 1.7129, "theoretical_loss": 4.422392707950824, "tokens_seen": 186449920 }, { "epoch": 0.06, "learning_rate": 0.0007624152457372117, "loss": 1.7146, "theoretical_loss": 4.42219738855824, "tokens_seen": 186515456 }, { "epoch": 0.06, "learning_rate": 0.0007623991975927784, "loss": 1.663, "theoretical_loss": 4.422002156991597, "tokens_seen": 186580992 }, { "epoch": 0.06, "learning_rate": 0.0007623831494483451, "loss": 1.6474, "theoretical_loss": 4.421807013180571, "tokens_seen": 186646528 }, { "epoch": 0.06, "learning_rate": 0.0007623671013039118, "loss": 1.7353, "theoretical_loss": 4.421611957054925, "tokens_seen": 186712064 }, { "epoch": 0.06, "objective/train/docs_used": 68808, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6501104831695557, "objective/train/theoretical_loss": 4.421416988544503, "objective/train/tokens_used": 207237600, "theoretical_loss": 4.421416988544503, "tokens_seen": 186777600 }, { "epoch": 0.06, "learning_rate": 0.0007623510531594785, "loss": 1.7163, "theoretical_loss": 4.421416988544503, "tokens_seen": 186777600 }, { "epoch": 0.06, "learning_rate": 0.0007623350050150452, "loss": 1.7814, "theoretical_loss": 4.4212221075792275, "tokens_seen": 186843136 }, { "epoch": 0.06, "learning_rate": 0.0007623189568706119, "loss": 1.7138, "theoretical_loss": 4.421027314089103, "tokens_seen": 186908672 }, { "epoch": 0.06, "learning_rate": 0.0007623029087261786, "loss": 1.6188, "theoretical_loss": 4.420832608004215, "tokens_seen": 186974208 }, { "epoch": 0.06, "learning_rate": 0.0007622868605817453, "loss": 1.6911, "theoretical_loss": 4.420637989254726, "tokens_seen": 187039744 }, { "epoch": 0.06, "learning_rate": 0.000762270812437312, "loss": 1.6176, "theoretical_loss": 4.420443457770885, "tokens_seen": 187105280 }, { "epoch": 0.06, "learning_rate": 0.0007622547642928787, "loss": 1.7182, "theoretical_loss": 4.420249013483014, "tokens_seen": 187170816 }, { "epoch": 0.06, "learning_rate": 0.0007622387161484454, "loss": 1.7003, "theoretical_loss": 4.42005465632152, "tokens_seen": 187236352 }, { "epoch": 0.06, "learning_rate": 0.0007622226680040121, "loss": 1.7183, "theoretical_loss": 4.419860386216888, "tokens_seen": 187301888 }, { "epoch": 0.06, "learning_rate": 0.0007622066198595788, "loss": 1.737, "theoretical_loss": 4.419666203099683, "tokens_seen": 187367424 }, { "epoch": 0.06, "learning_rate": 0.0007621905717151455, "loss": 1.7014, "theoretical_loss": 4.41947210690055, "tokens_seen": 187432960 }, { "epoch": 0.06, "learning_rate": 0.0007621745235707122, "loss": 1.7964, "theoretical_loss": 4.419278097550212, "tokens_seen": 187498496 }, { "epoch": 0.06, "learning_rate": 0.0007621584754262789, "loss": 1.6903, "theoretical_loss": 4.419084174979473, "tokens_seen": 187564032 }, { "epoch": 0.06, "learning_rate": 0.0007621424272818456, "loss": 1.7771, "theoretical_loss": 4.4188903391192165, "tokens_seen": 187629568 }, { "epoch": 0.06, "learning_rate": 0.0007621263791374123, "loss": 1.7877, "theoretical_loss": 4.418696589900405, "tokens_seen": 187695104 }, { "epoch": 0.06, "learning_rate": 0.000762110330992979, "loss": 1.7553, "theoretical_loss": 4.418502927254078, "tokens_seen": 187760640 }, { "epoch": 0.06, "learning_rate": 0.0007620942828485457, "loss": 1.7333, "theoretical_loss": 4.418309351111356, "tokens_seen": 187826176 }, { "epoch": 0.06, "learning_rate": 0.0007620782347041124, "loss": 1.6975, "theoretical_loss": 4.418115861403439, "tokens_seen": 187891712 }, { "epoch": 0.06, "learning_rate": 0.0007620621865596791, "loss": 1.5968, "theoretical_loss": 4.417922458061603, "tokens_seen": 187957248 }, { "epoch": 0.06, "learning_rate": 0.0007620461384152458, "loss": 1.6263, "theoretical_loss": 4.417729141017205, "tokens_seen": 188022784 }, { "epoch": 0.06, "learning_rate": 0.0007620300902708124, "loss": 1.7959, "theoretical_loss": 4.4175359102016785, "tokens_seen": 188088320 }, { "epoch": 0.06, "learning_rate": 0.0007620140421263792, "loss": 1.7892, "theoretical_loss": 4.417342765546538, "tokens_seen": 188153856 }, { "epoch": 0.06, "learning_rate": 0.0007619979939819458, "loss": 1.75, "theoretical_loss": 4.417149706983373, "tokens_seen": 188219392 }, { "epoch": 0.06, "learning_rate": 0.0007619819458375126, "loss": 1.6113, "theoretical_loss": 4.416956734443854, "tokens_seen": 188284928 }, { "epoch": 0.06, "learning_rate": 0.0007619658976930792, "loss": 1.5337, "theoretical_loss": 4.416763847859728, "tokens_seen": 188350464 }, { "epoch": 0.06, "learning_rate": 0.000761949849548646, "loss": 1.6184, "theoretical_loss": 4.41657104716282, "tokens_seen": 188416000 }, { "epoch": 0.06, "learning_rate": 0.0007619338014042126, "loss": 1.7843, "theoretical_loss": 4.416378332285031, "tokens_seen": 188481536 }, { "epoch": 0.06, "learning_rate": 0.0007619177532597794, "loss": 1.6608, "theoretical_loss": 4.4161857031583445, "tokens_seen": 188547072 }, { "epoch": 0.06, "learning_rate": 0.000761901705115346, "loss": 1.7039, "theoretical_loss": 4.415993159714818, "tokens_seen": 188612608 }, { "epoch": 0.06, "learning_rate": 0.0007618856569709127, "loss": 1.8617, "theoretical_loss": 4.415800701886584, "tokens_seen": 188678144 }, { "epoch": 0.06, "learning_rate": 0.0007618696088264794, "loss": 1.813, "theoretical_loss": 4.415608329605859, "tokens_seen": 188743680 }, { "epoch": 0.06, "learning_rate": 0.0007618535606820461, "loss": 1.5579, "theoretical_loss": 4.415416042804931, "tokens_seen": 188809216 }, { "epoch": 0.06, "learning_rate": 0.0007618375125376128, "loss": 1.6717, "theoretical_loss": 4.415223841416167, "tokens_seen": 188874752 }, { "epoch": 0.06, "learning_rate": 0.0007618214643931795, "loss": 1.7537, "theoretical_loss": 4.415031725372011, "tokens_seen": 188940288 }, { "epoch": 0.06, "learning_rate": 0.0007618054162487462, "loss": 1.6978, "theoretical_loss": 4.414839694604985, "tokens_seen": 189005824 }, { "epoch": 0.06, "learning_rate": 0.0007617893681043129, "loss": 1.6423, "theoretical_loss": 4.414647749047685, "tokens_seen": 189071360 }, { "epoch": 0.06, "learning_rate": 0.0007617733199598796, "loss": 1.6646, "theoretical_loss": 4.414455888632786, "tokens_seen": 189136896 }, { "epoch": 0.06, "learning_rate": 0.0007617572718154463, "loss": 1.7303, "theoretical_loss": 4.414264113293038, "tokens_seen": 189202432 }, { "epoch": 0.06, "learning_rate": 0.000761741223671013, "loss": 1.7068, "theoretical_loss": 4.41407242296127, "tokens_seen": 189267968 }, { "epoch": 0.06, "learning_rate": 0.0007617251755265797, "loss": 1.6995, "theoretical_loss": 4.413880817570382, "tokens_seen": 189333504 }, { "epoch": 0.06, "learning_rate": 0.0007617091273821465, "loss": 1.6645, "theoretical_loss": 4.413689297053356, "tokens_seen": 189399040 }, { "epoch": 0.06, "learning_rate": 0.0007616930792377131, "loss": 1.6716, "theoretical_loss": 4.413497861343247, "tokens_seen": 189464576 }, { "epoch": 0.06, "learning_rate": 0.0007616770310932799, "loss": 1.6816, "theoretical_loss": 4.413306510373185, "tokens_seen": 189530112 }, { "epoch": 0.06, "learning_rate": 0.0007616609829488465, "loss": 1.62, "theoretical_loss": 4.41311524407638, "tokens_seen": 189595648 }, { "epoch": 0.06, "learning_rate": 0.0007616449348044133, "loss": 1.715, "theoretical_loss": 4.412924062386112, "tokens_seen": 189661184 }, { "epoch": 0.06, "learning_rate": 0.0007616288866599799, "loss": 1.8238, "theoretical_loss": 4.412732965235742, "tokens_seen": 189726720 }, { "epoch": 0.06, "learning_rate": 0.0007616128385155467, "loss": 1.8357, "theoretical_loss": 4.412541952558703, "tokens_seen": 189792256 }, { "epoch": 0.06, "learning_rate": 0.0007615967903711133, "loss": 1.6682, "theoretical_loss": 4.412351024288504, "tokens_seen": 189857792 }, { "epoch": 0.06, "learning_rate": 0.0007615807422266801, "loss": 1.6815, "theoretical_loss": 4.412160180358731, "tokens_seen": 189923328 }, { "epoch": 0.06, "learning_rate": 0.0007615646940822467, "loss": 1.6328, "theoretical_loss": 4.411969420703042, "tokens_seen": 189988864 }, { "epoch": 0.06, "objective/train/docs_used": 70105, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.612324595451355, "objective/train/theoretical_loss": 4.411778745255174, "objective/train/tokens_used": 210514400, "theoretical_loss": 4.411778745255174, "tokens_seen": 190054400 }, { "epoch": 0.06, "learning_rate": 0.0007615486459378135, "loss": 1.6841, "theoretical_loss": 4.411778745255174, "tokens_seen": 190054400 }, { "epoch": 0.06, "learning_rate": 0.0007615325977933802, "loss": 1.7375, "theoretical_loss": 4.411588153948935, "tokens_seen": 190119936 }, { "epoch": 0.06, "learning_rate": 0.0007615165496489469, "loss": 1.8672, "theoretical_loss": 4.411397646718211, "tokens_seen": 190185472 }, { "epoch": 0.06, "learning_rate": 0.0007615005015045136, "loss": 1.6434, "theoretical_loss": 4.411207223496962, "tokens_seen": 190251008 }, { "epoch": 0.06, "learning_rate": 0.0007614844533600803, "loss": 1.6167, "theoretical_loss": 4.41101688421922, "tokens_seen": 190316544 }, { "epoch": 0.06, "learning_rate": 0.000761468405215647, "loss": 1.7425, "theoretical_loss": 4.410826628819094, "tokens_seen": 190382080 }, { "epoch": 0.06, "learning_rate": 0.0007614523570712137, "loss": 1.6488, "theoretical_loss": 4.410636457230767, "tokens_seen": 190447616 }, { "epoch": 0.06, "learning_rate": 0.0007614363089267804, "loss": 1.6163, "theoretical_loss": 4.410446369388497, "tokens_seen": 190513152 }, { "epoch": 0.06, "learning_rate": 0.0007614202607823471, "loss": 1.7284, "theoretical_loss": 4.410256365226614, "tokens_seen": 190578688 }, { "epoch": 0.06, "learning_rate": 0.0007614042126379138, "loss": 1.7268, "theoretical_loss": 4.410066444679524, "tokens_seen": 190644224 }, { "epoch": 0.06, "learning_rate": 0.0007613881644934805, "loss": 1.7091, "theoretical_loss": 4.409876607681706, "tokens_seen": 190709760 }, { "epoch": 0.06, "learning_rate": 0.0007613721163490472, "loss": 1.5607, "theoretical_loss": 4.409686854167714, "tokens_seen": 190775296 }, { "epoch": 0.06, "learning_rate": 0.0007613560682046139, "loss": 1.6649, "theoretical_loss": 4.4094971840721735, "tokens_seen": 190840832 }, { "epoch": 0.06, "learning_rate": 0.0007613400200601806, "loss": 1.7259, "theoretical_loss": 4.409307597329786, "tokens_seen": 190906368 }, { "epoch": 0.06, "learning_rate": 0.0007613239719157473, "loss": 1.8349, "theoretical_loss": 4.409118093875324, "tokens_seen": 190971904 }, { "epoch": 0.06, "learning_rate": 0.000761307923771314, "loss": 1.7556, "theoretical_loss": 4.408928673643636, "tokens_seen": 191037440 }, { "epoch": 0.06, "learning_rate": 0.0007612918756268807, "loss": 1.8219, "theoretical_loss": 4.408739336569644, "tokens_seen": 191102976 }, { "epoch": 0.06, "learning_rate": 0.0007612758274824474, "loss": 1.5849, "theoretical_loss": 4.408550082588339, "tokens_seen": 191168512 }, { "epoch": 0.06, "learning_rate": 0.0007612597793380141, "loss": 1.716, "theoretical_loss": 4.408360911634789, "tokens_seen": 191234048 }, { "epoch": 0.06, "learning_rate": 0.0007612437311935808, "loss": 1.6816, "theoretical_loss": 4.408171823644135, "tokens_seen": 191299584 }, { "epoch": 0.06, "learning_rate": 0.0007612276830491475, "loss": 1.7805, "theoretical_loss": 4.407982818551589, "tokens_seen": 191365120 }, { "epoch": 0.06, "learning_rate": 0.0007612116349047142, "loss": 1.822, "theoretical_loss": 4.407793896292437, "tokens_seen": 191430656 }, { "epoch": 0.06, "learning_rate": 0.0007611955867602809, "loss": 1.7945, "theoretical_loss": 4.407605056802035, "tokens_seen": 191496192 }, { "epoch": 0.06, "learning_rate": 0.0007611795386158476, "loss": 1.7431, "theoretical_loss": 4.407416300015816, "tokens_seen": 191561728 }, { "epoch": 0.06, "learning_rate": 0.0007611634904714143, "loss": 1.562, "theoretical_loss": 4.407227625869283, "tokens_seen": 191627264 }, { "epoch": 0.06, "learning_rate": 0.000761147442326981, "loss": 1.7065, "theoretical_loss": 4.407039034298011, "tokens_seen": 191692800 }, { "epoch": 0.06, "learning_rate": 0.0007611313941825477, "loss": 1.55, "theoretical_loss": 4.4068505252376475, "tokens_seen": 191758336 }, { "epoch": 0.06, "learning_rate": 0.0007611153460381144, "loss": 1.8253, "theoretical_loss": 4.406662098623913, "tokens_seen": 191823872 }, { "epoch": 0.06, "learning_rate": 0.0007610992978936811, "loss": 1.5841, "theoretical_loss": 4.406473754392598, "tokens_seen": 191889408 }, { "epoch": 0.06, "learning_rate": 0.0007610832497492478, "loss": 1.7049, "theoretical_loss": 4.406285492479569, "tokens_seen": 191954944 }, { "epoch": 0.06, "learning_rate": 0.0007610672016048145, "loss": 1.7289, "theoretical_loss": 4.406097312820759, "tokens_seen": 192020480 }, { "epoch": 0.06, "learning_rate": 0.0007610511534603812, "loss": 1.5932, "theoretical_loss": 4.4059092153521755, "tokens_seen": 192086016 }, { "epoch": 0.06, "learning_rate": 0.0007610351053159479, "loss": 1.7163, "theoretical_loss": 4.405721200009898, "tokens_seen": 192151552 }, { "epoch": 0.06, "learning_rate": 0.0007610190571715146, "loss": 1.7145, "theoretical_loss": 4.405533266730077, "tokens_seen": 192217088 }, { "epoch": 0.06, "learning_rate": 0.0007610030090270813, "loss": 1.7585, "theoretical_loss": 4.405345415448934, "tokens_seen": 192282624 }, { "epoch": 0.06, "learning_rate": 0.000760986960882648, "loss": 1.809, "theoretical_loss": 4.405157646102762, "tokens_seen": 192348160 }, { "epoch": 0.06, "learning_rate": 0.0007609709127382147, "loss": 1.6969, "theoretical_loss": 4.404969958627927, "tokens_seen": 192413696 }, { "epoch": 0.06, "learning_rate": 0.0007609548645937814, "loss": 1.6836, "theoretical_loss": 4.4047823529608605, "tokens_seen": 192479232 }, { "epoch": 0.06, "learning_rate": 0.0007609388164493481, "loss": 1.6517, "theoretical_loss": 4.404594829038071, "tokens_seen": 192544768 }, { "epoch": 0.06, "learning_rate": 0.0007609227683049148, "loss": 1.8073, "theoretical_loss": 4.404407386796137, "tokens_seen": 192610304 }, { "epoch": 0.06, "learning_rate": 0.0007609067201604815, "loss": 1.6976, "theoretical_loss": 4.404220026171704, "tokens_seen": 192675840 }, { "epoch": 0.06, "learning_rate": 0.0007608906720160482, "loss": 1.8556, "theoretical_loss": 4.404032747101493, "tokens_seen": 192741376 }, { "epoch": 0.06, "learning_rate": 0.0007608746238716149, "loss": 1.6249, "theoretical_loss": 4.4038455495222895, "tokens_seen": 192806912 }, { "epoch": 0.06, "learning_rate": 0.0007608585757271816, "loss": 1.7009, "theoretical_loss": 4.403658433370957, "tokens_seen": 192872448 }, { "epoch": 0.06, "learning_rate": 0.0007608425275827483, "loss": 1.6373, "theoretical_loss": 4.403471398584422, "tokens_seen": 192937984 }, { "epoch": 0.06, "learning_rate": 0.000760826479438315, "loss": 1.6357, "theoretical_loss": 4.403284445099688, "tokens_seen": 193003520 }, { "epoch": 0.06, "learning_rate": 0.0007608104312938817, "loss": 1.7354, "theoretical_loss": 4.403097572853823, "tokens_seen": 193069056 }, { "epoch": 0.06, "learning_rate": 0.0007607943831494484, "loss": 1.7761, "theoretical_loss": 4.402910781783968, "tokens_seen": 193134592 }, { "epoch": 0.06, "learning_rate": 0.0007607783350050151, "loss": 1.7687, "theoretical_loss": 4.4027240718273335, "tokens_seen": 193200128 }, { "epoch": 0.06, "learning_rate": 0.0007607622868605818, "loss": 1.6265, "theoretical_loss": 4.4025374429212, "tokens_seen": 193265664 }, { "epoch": 0.06, "objective/train/docs_used": 71032, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6292566061019897, "objective/train/theoretical_loss": 4.402350895002916, "objective/train/tokens_used": 213791200, "theoretical_loss": 4.402350895002916, "tokens_seen": 193331200 }, { "epoch": 0.06, "learning_rate": 0.0007607462387161485, "loss": 1.6732, "theoretical_loss": 4.402350895002916, "tokens_seen": 193331200 }, { "epoch": 0.06, "learning_rate": 0.0007607301905717152, "loss": 1.7352, "theoretical_loss": 4.402164428009904, "tokens_seen": 193396736 }, { "epoch": 0.06, "learning_rate": 0.0007607141424272819, "loss": 1.7918, "theoretical_loss": 4.40197804187965, "tokens_seen": 193462272 }, { "epoch": 0.06, "learning_rate": 0.0007606980942828486, "loss": 1.6467, "theoretical_loss": 4.401791736549714, "tokens_seen": 193527808 }, { "epoch": 0.06, "learning_rate": 0.0007606820461384153, "loss": 1.7642, "theoretical_loss": 4.401605511957724, "tokens_seen": 193593344 }, { "epoch": 0.06, "learning_rate": 0.000760665997993982, "loss": 1.6585, "theoretical_loss": 4.401419368041377, "tokens_seen": 193658880 }, { "epoch": 0.06, "learning_rate": 0.0007606499498495487, "loss": 1.6439, "theoretical_loss": 4.401233304738438, "tokens_seen": 193724416 }, { "epoch": 0.06, "learning_rate": 0.0007606339017051153, "loss": 1.7171, "theoretical_loss": 4.401047321986745, "tokens_seen": 193789952 }, { "epoch": 0.06, "learning_rate": 0.0007606178535606821, "loss": 1.7524, "theoretical_loss": 4.4008614197242, "tokens_seen": 193855488 }, { "epoch": 0.06, "learning_rate": 0.0007606018054162487, "loss": 1.7401, "theoretical_loss": 4.400675597888777, "tokens_seen": 193921024 }, { "epoch": 0.06, "learning_rate": 0.0007605857572718155, "loss": 1.6179, "theoretical_loss": 4.400489856418518, "tokens_seen": 193986560 }, { "epoch": 0.06, "learning_rate": 0.0007605697091273821, "loss": 1.6743, "theoretical_loss": 4.4003041952515325, "tokens_seen": 194052096 }, { "epoch": 0.06, "learning_rate": 0.0007605536609829489, "loss": 1.7264, "theoretical_loss": 4.400118614326001, "tokens_seen": 194117632 }, { "epoch": 0.06, "learning_rate": 0.0007605376128385155, "loss": 1.6531, "theoretical_loss": 4.39993311358017, "tokens_seen": 194183168 }, { "epoch": 0.06, "learning_rate": 0.0007605215646940823, "loss": 1.5623, "theoretical_loss": 4.399747692952355, "tokens_seen": 194248704 }, { "epoch": 0.06, "learning_rate": 0.0007605055165496489, "loss": 1.7761, "theoretical_loss": 4.399562352380941, "tokens_seen": 194314240 }, { "epoch": 0.06, "learning_rate": 0.0007604894684052156, "loss": 1.7038, "theoretical_loss": 4.399377091804379, "tokens_seen": 194379776 }, { "epoch": 0.06, "learning_rate": 0.0007604734202607823, "loss": 1.7686, "theoretical_loss": 4.399191911161191, "tokens_seen": 194445312 }, { "epoch": 0.06, "learning_rate": 0.000760457372116349, "loss": 1.6672, "theoretical_loss": 4.399006810389963, "tokens_seen": 194510848 }, { "epoch": 0.06, "learning_rate": 0.0007604413239719157, "loss": 1.6675, "theoretical_loss": 4.398821789429354, "tokens_seen": 194576384 }, { "epoch": 0.06, "learning_rate": 0.0007604252758274824, "loss": 1.844, "theoretical_loss": 4.398636848218084, "tokens_seen": 194641920 }, { "epoch": 0.06, "learning_rate": 0.0007604092276830491, "loss": 1.7601, "theoretical_loss": 4.398451986694948, "tokens_seen": 194707456 }, { "epoch": 0.06, "learning_rate": 0.0007603931795386158, "loss": 1.7763, "theoretical_loss": 4.398267204798803, "tokens_seen": 194772992 }, { "epoch": 0.06, "learning_rate": 0.0007603771313941825, "loss": 1.6735, "theoretical_loss": 4.398082502468577, "tokens_seen": 194838528 }, { "epoch": 0.06, "learning_rate": 0.0007603610832497492, "loss": 1.5872, "theoretical_loss": 4.397897879643262, "tokens_seen": 194904064 }, { "epoch": 0.06, "learning_rate": 0.000760345035105316, "loss": 1.6719, "theoretical_loss": 4.39771333626192, "tokens_seen": 194969600 }, { "epoch": 0.06, "learning_rate": 0.0007603289869608827, "loss": 1.7731, "theoretical_loss": 4.397528872263679, "tokens_seen": 195035136 }, { "epoch": 0.06, "learning_rate": 0.0007603129388164494, "loss": 1.6565, "theoretical_loss": 4.397344487587736, "tokens_seen": 195100672 }, { "epoch": 0.06, "learning_rate": 0.0007602968906720161, "loss": 1.7334, "theoretical_loss": 4.3971601821733515, "tokens_seen": 195166208 }, { "epoch": 0.06, "learning_rate": 0.0007602808425275828, "loss": 1.6964, "theoretical_loss": 4.396975955959856, "tokens_seen": 195231744 }, { "epoch": 0.06, "learning_rate": 0.0007602647943831495, "loss": 1.7469, "theoretical_loss": 4.396791808886643, "tokens_seen": 195297280 }, { "epoch": 0.06, "learning_rate": 0.0007602487462387162, "loss": 1.8126, "theoretical_loss": 4.396607740893179, "tokens_seen": 195362816 }, { "epoch": 0.06, "learning_rate": 0.0007602326980942829, "loss": 1.6569, "theoretical_loss": 4.396423751918991, "tokens_seen": 195428352 }, { "epoch": 0.06, "learning_rate": 0.0007602166499498496, "loss": 1.5221, "theoretical_loss": 4.396239841903674, "tokens_seen": 195493888 }, { "epoch": 0.06, "learning_rate": 0.0007602006018054163, "loss": 1.6202, "theoretical_loss": 4.396056010786893, "tokens_seen": 195559424 }, { "epoch": 0.06, "learning_rate": 0.000760184553660983, "loss": 1.8545, "theoretical_loss": 4.3958722585083745, "tokens_seen": 195624960 }, { "epoch": 0.06, "learning_rate": 0.0007601685055165497, "loss": 1.7108, "theoretical_loss": 4.395688585007913, "tokens_seen": 195690496 }, { "epoch": 0.06, "learning_rate": 0.0007601524573721164, "loss": 1.7769, "theoretical_loss": 4.395504990225371, "tokens_seen": 195756032 }, { "epoch": 0.06, "learning_rate": 0.0007601364092276831, "loss": 1.8607, "theoretical_loss": 4.395321474100673, "tokens_seen": 195821568 }, { "epoch": 0.06, "learning_rate": 0.0007601203610832498, "loss": 1.7631, "theoretical_loss": 4.395138036573814, "tokens_seen": 195887104 }, { "epoch": 0.06, "learning_rate": 0.0007601043129388165, "loss": 1.7287, "theoretical_loss": 4.3949546775848525, "tokens_seen": 195952640 }, { "epoch": 0.06, "learning_rate": 0.0007600882647943832, "loss": 1.6529, "theoretical_loss": 4.394771397073912, "tokens_seen": 196018176 }, { "epoch": 0.06, "learning_rate": 0.0007600722166499499, "loss": 1.7395, "theoretical_loss": 4.394588194981182, "tokens_seen": 196083712 }, { "epoch": 0.06, "learning_rate": 0.0007600561685055166, "loss": 1.8479, "theoretical_loss": 4.394405071246919, "tokens_seen": 196149248 }, { "epoch": 0.06, "learning_rate": 0.0007600401203610833, "loss": 1.733, "theoretical_loss": 4.394222025811445, "tokens_seen": 196214784 }, { "epoch": 0.06, "learning_rate": 0.00076002407221665, "loss": 1.6186, "theoretical_loss": 4.394039058615144, "tokens_seen": 196280320 }, { "epoch": 0.06, "learning_rate": 0.0007600080240722167, "loss": 1.6977, "theoretical_loss": 4.39385616959847, "tokens_seen": 196345856 }, { "epoch": 0.06, "learning_rate": 0.0007599919759277834, "loss": 1.7501, "theoretical_loss": 4.393673358701939, "tokens_seen": 196411392 }, { "epoch": 0.06, "learning_rate": 0.0007599759277833501, "loss": 1.5986, "theoretical_loss": 4.393490625866132, "tokens_seen": 196476928 }, { "epoch": 0.06, "learning_rate": 0.0007599598796389168, "loss": 1.777, "theoretical_loss": 4.393307971031697, "tokens_seen": 196542464 }, { "epoch": 0.06, "objective/train/docs_used": 71932, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.4793413877487183, "objective/train/theoretical_loss": 4.3931253941393456, "objective/train/tokens_used": 217068000, "theoretical_loss": 4.3931253941393456, "tokens_seen": 196608000 }, { "epoch": 0.06, "learning_rate": 0.0007599438314944835, "loss": 1.5309, "theoretical_loss": 4.3931253941393456, "tokens_seen": 196608000 }, { "epoch": 0.06, "learning_rate": 0.0007599277833500502, "loss": 1.7435, "theoretical_loss": 4.392942895129854, "tokens_seen": 196673536 }, { "epoch": 0.06, "learning_rate": 0.0007599117352056169, "loss": 1.6521, "theoretical_loss": 4.392760473944064, "tokens_seen": 196739072 }, { "epoch": 0.06, "learning_rate": 0.0007598956870611836, "loss": 1.708, "theoretical_loss": 4.392578130522883, "tokens_seen": 196804608 }, { "epoch": 0.06, "learning_rate": 0.0007598796389167503, "loss": 1.7905, "theoretical_loss": 4.392395864807279, "tokens_seen": 196870144 }, { "epoch": 0.06, "learning_rate": 0.000759863590772317, "loss": 1.7152, "theoretical_loss": 4.392213676738288, "tokens_seen": 196935680 }, { "epoch": 0.06, "learning_rate": 0.0007598475426278837, "loss": 1.8214, "theoretical_loss": 4.3920315662570095, "tokens_seen": 197001216 }, { "epoch": 0.06, "learning_rate": 0.0007598314944834504, "loss": 1.7028, "theoretical_loss": 4.391849533304607, "tokens_seen": 197066752 }, { "epoch": 0.06, "learning_rate": 0.0007598154463390171, "loss": 1.6381, "theoretical_loss": 4.391667577822308, "tokens_seen": 197132288 }, { "epoch": 0.06, "learning_rate": 0.0007597993981945838, "loss": 1.7244, "theoretical_loss": 4.391485699751406, "tokens_seen": 197197824 }, { "epoch": 0.06, "learning_rate": 0.0007597833500501505, "loss": 1.7536, "theoretical_loss": 4.391303899033253, "tokens_seen": 197263360 }, { "epoch": 0.06, "learning_rate": 0.0007597673019057172, "loss": 1.6795, "theoretical_loss": 4.391122175609272, "tokens_seen": 197328896 }, { "epoch": 0.06, "learning_rate": 0.0007597512537612839, "loss": 1.6457, "theoretical_loss": 4.390940529420946, "tokens_seen": 197394432 }, { "epoch": 0.06, "learning_rate": 0.0007597352056168506, "loss": 1.6911, "theoretical_loss": 4.39075896040982, "tokens_seen": 197459968 }, { "epoch": 0.06, "learning_rate": 0.0007597191574724173, "loss": 1.5381, "theoretical_loss": 4.390577468517508, "tokens_seen": 197525504 }, { "epoch": 0.06, "learning_rate": 0.000759703109327984, "loss": 1.7558, "theoretical_loss": 4.390396053685682, "tokens_seen": 197591040 }, { "epoch": 0.06, "learning_rate": 0.0007596870611835507, "loss": 1.7281, "theoretical_loss": 4.39021471585608, "tokens_seen": 197656576 }, { "epoch": 0.06, "learning_rate": 0.0007596710130391174, "loss": 1.6057, "theoretical_loss": 4.390033454970504, "tokens_seen": 197722112 }, { "epoch": 0.06, "learning_rate": 0.0007596549648946841, "loss": 1.6983, "theoretical_loss": 4.389852270970818, "tokens_seen": 197787648 }, { "epoch": 0.06, "learning_rate": 0.0007596389167502508, "loss": 1.7131, "theoretical_loss": 4.389671163798949, "tokens_seen": 197853184 }, { "epoch": 0.06, "learning_rate": 0.0007596228686058175, "loss": 1.7804, "theoretical_loss": 4.3894901333968885, "tokens_seen": 197918720 }, { "epoch": 0.06, "learning_rate": 0.0007596068204613842, "loss": 1.7999, "theoretical_loss": 4.38930917970669, "tokens_seen": 197984256 }, { "epoch": 0.06, "learning_rate": 0.0007595907723169509, "loss": 1.7063, "theoretical_loss": 4.3891283026704695, "tokens_seen": 198049792 }, { "epoch": 0.06, "learning_rate": 0.0007595747241725176, "loss": 1.7568, "theoretical_loss": 4.388947502230407, "tokens_seen": 198115328 }, { "epoch": 0.06, "learning_rate": 0.0007595586760280843, "loss": 1.6771, "theoretical_loss": 4.388766778328744, "tokens_seen": 198180864 }, { "epoch": 0.06, "learning_rate": 0.000759542627883651, "loss": 1.6316, "theoretical_loss": 4.388586130907786, "tokens_seen": 198246400 }, { "epoch": 0.06, "learning_rate": 0.0007595265797392177, "loss": 1.7721, "theoretical_loss": 4.388405559909899, "tokens_seen": 198311936 }, { "epoch": 0.06, "learning_rate": 0.0007595105315947844, "loss": 1.672, "theoretical_loss": 4.3882250652775125, "tokens_seen": 198377472 }, { "epoch": 0.06, "learning_rate": 0.0007594944834503511, "loss": 1.6478, "theoretical_loss": 4.38804464695312, "tokens_seen": 198443008 }, { "epoch": 0.06, "learning_rate": 0.0007594784353059178, "loss": 1.648, "theoretical_loss": 4.387864304879275, "tokens_seen": 198508544 }, { "epoch": 0.06, "learning_rate": 0.0007594623871614845, "loss": 1.7571, "theoretical_loss": 4.387684038998595, "tokens_seen": 198574080 }, { "epoch": 0.06, "learning_rate": 0.0007594463390170512, "loss": 1.7198, "theoretical_loss": 4.387503849253757, "tokens_seen": 198639616 }, { "epoch": 0.06, "learning_rate": 0.0007594302908726179, "loss": 1.7731, "theoretical_loss": 4.387323735587502, "tokens_seen": 198705152 }, { "epoch": 0.06, "learning_rate": 0.0007594142427281846, "loss": 1.7331, "theoretical_loss": 4.387143697942634, "tokens_seen": 198770688 }, { "epoch": 0.06, "learning_rate": 0.0007593981945837513, "loss": 1.8219, "theoretical_loss": 4.386963736262016, "tokens_seen": 198836224 }, { "epoch": 0.06, "learning_rate": 0.000759382146439318, "loss": 1.685, "theoretical_loss": 4.386783850488575, "tokens_seen": 198901760 }, { "epoch": 0.06, "learning_rate": 0.0007593660982948847, "loss": 1.7198, "theoretical_loss": 4.386604040565299, "tokens_seen": 198967296 }, { "epoch": 0.06, "learning_rate": 0.0007593500501504514, "loss": 1.7957, "theoretical_loss": 4.386424306435236, "tokens_seen": 199032832 }, { "epoch": 0.06, "learning_rate": 0.0007593340020060181, "loss": 1.7063, "theoretical_loss": 4.386244648041498, "tokens_seen": 199098368 }, { "epoch": 0.06, "learning_rate": 0.0007593179538615848, "loss": 1.6459, "theoretical_loss": 4.386065065327258, "tokens_seen": 199163904 }, { "epoch": 0.06, "learning_rate": 0.0007593019057171515, "loss": 1.6441, "theoretical_loss": 4.385885558235747, "tokens_seen": 199229440 }, { "epoch": 0.06, "learning_rate": 0.0007592858575727182, "loss": 1.7275, "theoretical_loss": 4.385706126710264, "tokens_seen": 199294976 }, { "epoch": 0.06, "learning_rate": 0.000759269809428285, "loss": 1.7563, "theoretical_loss": 4.385526770694161, "tokens_seen": 199360512 }, { "epoch": 0.06, "learning_rate": 0.0007592537612838516, "loss": 1.6168, "theoretical_loss": 4.385347490130858, "tokens_seen": 199426048 }, { "epoch": 0.06, "learning_rate": 0.0007592377131394182, "loss": 1.678, "theoretical_loss": 4.385168284963832, "tokens_seen": 199491584 }, { "epoch": 0.06, "learning_rate": 0.000759221664994985, "loss": 1.7211, "theoretical_loss": 4.384989155136621, "tokens_seen": 199557120 }, { "epoch": 0.06, "learning_rate": 0.0007592056168505516, "loss": 1.6404, "theoretical_loss": 4.384810100592826, "tokens_seen": 199622656 }, { "epoch": 0.06, "learning_rate": 0.0007591895687061184, "loss": 1.7634, "theoretical_loss": 4.384631121276108, "tokens_seen": 199688192 }, { "epoch": 0.06, "learning_rate": 0.000759173520561685, "loss": 1.649, "theoretical_loss": 4.384452217130187, "tokens_seen": 199753728 }, { "epoch": 0.06, "learning_rate": 0.0007591574724172518, "loss": 1.6701, "theoretical_loss": 4.384273388098845, "tokens_seen": 199819264 }, { "epoch": 0.06, "objective/train/docs_used": 73160, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7228140830993652, "objective/train/theoretical_loss": 4.384094634125924, "objective/train/tokens_used": 220344800, "theoretical_loss": 4.384094634125924, "tokens_seen": 199884800 }, { "epoch": 0.06, "learning_rate": 0.0007591414242728184, "loss": 1.6614, "theoretical_loss": 4.384094634125924, "tokens_seen": 199884800 }, { "epoch": 0.06, "learning_rate": 0.0007591253761283852, "loss": 1.6742, "theoretical_loss": 4.383915955155326, "tokens_seen": 199950336 }, { "epoch": 0.06, "learning_rate": 0.0007591093279839518, "loss": 1.6602, "theoretical_loss": 4.383737351131016, "tokens_seen": 200015872 }, { "epoch": 0.06, "learning_rate": 0.0007590932798395186, "loss": 1.7915, "theoretical_loss": 4.383558821997014, "tokens_seen": 200081408 }, { "epoch": 0.06, "learning_rate": 0.0007590772316950852, "loss": 1.7205, "theoretical_loss": 4.383380367697406, "tokens_seen": 200146944 }, { "epoch": 0.06, "learning_rate": 0.000759061183550652, "loss": 1.7392, "theoretical_loss": 4.383201988176333, "tokens_seen": 200212480 }, { "epoch": 0.06, "learning_rate": 0.0007590451354062186, "loss": 1.735, "theoretical_loss": 4.383023683377999, "tokens_seen": 200278016 }, { "epoch": 0.06, "learning_rate": 0.0007590290872617854, "loss": 1.796, "theoretical_loss": 4.382845453246668, "tokens_seen": 200343552 }, { "epoch": 0.06, "learning_rate": 0.000759013039117352, "loss": 1.705, "theoretical_loss": 4.382667297726661, "tokens_seen": 200409088 }, { "epoch": 0.06, "learning_rate": 0.0007589969909729188, "loss": 1.6234, "theoretical_loss": 4.3824892167623615, "tokens_seen": 200474624 }, { "epoch": 0.06, "learning_rate": 0.0007589809428284855, "loss": 1.725, "theoretical_loss": 4.382311210298212, "tokens_seen": 200540160 }, { "epoch": 0.06, "learning_rate": 0.0007589648946840522, "loss": 1.7048, "theoretical_loss": 4.3821332782787135, "tokens_seen": 200605696 }, { "epoch": 0.06, "learning_rate": 0.0007589488465396189, "loss": 1.718, "theoretical_loss": 4.381955420648428, "tokens_seen": 200671232 }, { "epoch": 0.06, "learning_rate": 0.0007589327983951856, "loss": 1.7411, "theoretical_loss": 4.381777637351974, "tokens_seen": 200736768 }, { "epoch": 0.06, "learning_rate": 0.0007589167502507523, "loss": 1.8566, "theoretical_loss": 4.381599928334033, "tokens_seen": 200802304 }, { "epoch": 0.06, "learning_rate": 0.000758900702106319, "loss": 1.7141, "theoretical_loss": 4.381422293539345, "tokens_seen": 200867840 }, { "epoch": 0.06, "learning_rate": 0.0007588846539618857, "loss": 1.695, "theoretical_loss": 4.381244732912706, "tokens_seen": 200933376 }, { "epoch": 0.06, "learning_rate": 0.0007588686058174524, "loss": 1.6169, "theoretical_loss": 4.381067246398975, "tokens_seen": 200998912 }, { "epoch": 0.06, "learning_rate": 0.0007588525576730191, "loss": 1.6962, "theoretical_loss": 4.380889833943066, "tokens_seen": 201064448 }, { "epoch": 0.06, "learning_rate": 0.0007588365095285858, "loss": 1.6986, "theoretical_loss": 4.380712495489957, "tokens_seen": 201129984 }, { "epoch": 0.06, "learning_rate": 0.0007588204613841525, "loss": 1.7617, "theoretical_loss": 4.380535230984679, "tokens_seen": 201195520 }, { "epoch": 0.06, "learning_rate": 0.0007588044132397192, "loss": 1.6525, "theoretical_loss": 4.380358040372325, "tokens_seen": 201261056 }, { "epoch": 0.06, "learning_rate": 0.0007587883650952859, "loss": 1.7725, "theoretical_loss": 4.380180923598047, "tokens_seen": 201326592 }, { "epoch": 0.06, "learning_rate": 0.0007587723169508526, "loss": 1.7453, "theoretical_loss": 4.380003880607055, "tokens_seen": 201392128 }, { "epoch": 0.06, "learning_rate": 0.0007587562688064193, "loss": 1.787, "theoretical_loss": 4.379826911344617, "tokens_seen": 201457664 }, { "epoch": 0.06, "learning_rate": 0.000758740220661986, "loss": 1.7013, "theoretical_loss": 4.3796500157560585, "tokens_seen": 201523200 }, { "epoch": 0.06, "learning_rate": 0.0007587241725175527, "loss": 1.7672, "theoretical_loss": 4.379473193786765, "tokens_seen": 201588736 }, { "epoch": 0.06, "learning_rate": 0.0007587081243731194, "loss": 1.7227, "theoretical_loss": 4.37929644538218, "tokens_seen": 201654272 }, { "epoch": 0.06, "learning_rate": 0.0007586920762286861, "loss": 1.7301, "theoretical_loss": 4.379119770487805, "tokens_seen": 201719808 }, { "epoch": 0.06, "learning_rate": 0.0007586760280842528, "loss": 1.7752, "theoretical_loss": 4.378943169049198, "tokens_seen": 201785344 }, { "epoch": 0.06, "learning_rate": 0.0007586599799398195, "loss": 1.7993, "theoretical_loss": 4.378766641011978, "tokens_seen": 201850880 }, { "epoch": 0.06, "learning_rate": 0.0007586439317953862, "loss": 1.6976, "theoretical_loss": 4.378590186321819, "tokens_seen": 201916416 }, { "epoch": 0.06, "learning_rate": 0.0007586278836509529, "loss": 1.73, "theoretical_loss": 4.378413804924454, "tokens_seen": 201981952 }, { "epoch": 0.06, "learning_rate": 0.0007586118355065196, "loss": 1.6686, "theoretical_loss": 4.378237496765674, "tokens_seen": 202047488 }, { "epoch": 0.06, "learning_rate": 0.0007585957873620863, "loss": 1.6747, "theoretical_loss": 4.378061261791328, "tokens_seen": 202113024 }, { "epoch": 0.06, "learning_rate": 0.000758579739217653, "loss": 1.7438, "theoretical_loss": 4.377885099947321, "tokens_seen": 202178560 }, { "epoch": 0.06, "learning_rate": 0.0007585636910732197, "loss": 1.6094, "theoretical_loss": 4.3777090111796175, "tokens_seen": 202244096 }, { "epoch": 0.06, "learning_rate": 0.0007585476429287864, "loss": 1.7414, "theoretical_loss": 4.377532995434238, "tokens_seen": 202309632 }, { "epoch": 0.06, "learning_rate": 0.0007585315947843531, "loss": 1.7977, "theoretical_loss": 4.37735705265726, "tokens_seen": 202375168 }, { "epoch": 0.06, "learning_rate": 0.0007585155466399198, "loss": 1.6395, "theoretical_loss": 4.3771811827948195, "tokens_seen": 202440704 }, { "epoch": 0.06, "learning_rate": 0.0007584994984954865, "loss": 1.7897, "theoretical_loss": 4.37700538579311, "tokens_seen": 202506240 }, { "epoch": 0.06, "learning_rate": 0.0007584834503510532, "loss": 1.6584, "theoretical_loss": 4.37682966159838, "tokens_seen": 202571776 }, { "epoch": 0.06, "learning_rate": 0.0007584674022066199, "loss": 1.6479, "theoretical_loss": 4.376654010156939, "tokens_seen": 202637312 }, { "epoch": 0.06, "learning_rate": 0.0007584513540621866, "loss": 1.6911, "theoretical_loss": 4.376478431415148, "tokens_seen": 202702848 }, { "epoch": 0.06, "learning_rate": 0.0007584353059177533, "loss": 1.6819, "theoretical_loss": 4.376302925319427, "tokens_seen": 202768384 }, { "epoch": 0.06, "learning_rate": 0.00075841925777332, "loss": 1.7279, "theoretical_loss": 4.3761274918162565, "tokens_seen": 202833920 }, { "epoch": 0.06, "learning_rate": 0.0007584032096288867, "loss": 1.5278, "theoretical_loss": 4.375952130852169, "tokens_seen": 202899456 }, { "epoch": 0.06, "learning_rate": 0.0007583871614844534, "loss": 1.5301, "theoretical_loss": 4.375776842373755, "tokens_seen": 202964992 }, { "epoch": 0.06, "learning_rate": 0.0007583711133400201, "loss": 1.7144, "theoretical_loss": 4.375601626327662, "tokens_seen": 203030528 }, { "epoch": 0.06, "learning_rate": 0.0007583550651955868, "loss": 1.6743, "theoretical_loss": 4.375426482660595, "tokens_seen": 203096064 }, { "epoch": 0.06, "objective/train/docs_used": 74031, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6785008907318115, "objective/train/theoretical_loss": 4.375251411319312, "objective/train/tokens_used": 223621600, "theoretical_loss": 4.375251411319312, "tokens_seen": 203161600 }, { "epoch": 0.06, "learning_rate": 0.0007583390170511535, "loss": 1.7139, "theoretical_loss": 4.375251411319312, "tokens_seen": 203161600 }, { "epoch": 0.06, "learning_rate": 0.0007583229689067202, "loss": 1.6417, "theoretical_loss": 4.375076412250632, "tokens_seen": 203227136 }, { "epoch": 0.06, "learning_rate": 0.0007583069207622869, "loss": 1.6601, "theoretical_loss": 4.374901485401426, "tokens_seen": 203292672 }, { "epoch": 0.06, "learning_rate": 0.0007582908726178536, "loss": 1.6632, "theoretical_loss": 4.374726630718624, "tokens_seen": 203358208 }, { "epoch": 0.06, "learning_rate": 0.0007582748244734203, "loss": 1.6369, "theoretical_loss": 4.37455184814921, "tokens_seen": 203423744 }, { "epoch": 0.06, "learning_rate": 0.000758258776328987, "loss": 1.5944, "theoretical_loss": 4.374377137640225, "tokens_seen": 203489280 }, { "epoch": 0.06, "learning_rate": 0.0007582427281845537, "loss": 1.633, "theoretical_loss": 4.374202499138768, "tokens_seen": 203554816 }, { "epoch": 0.06, "learning_rate": 0.0007582266800401204, "loss": 1.6212, "theoretical_loss": 4.374027932591989, "tokens_seen": 203620352 }, { "epoch": 0.06, "learning_rate": 0.0007582106318956871, "loss": 1.6938, "theoretical_loss": 4.373853437947099, "tokens_seen": 203685888 }, { "epoch": 0.06, "learning_rate": 0.0007581945837512538, "loss": 1.7528, "theoretical_loss": 4.37367901515136, "tokens_seen": 203751424 }, { "epoch": 0.06, "learning_rate": 0.0007581785356068205, "loss": 1.7577, "theoretical_loss": 4.373504664152094, "tokens_seen": 203816960 }, { "epoch": 0.06, "learning_rate": 0.0007581624874623872, "loss": 1.5288, "theoretical_loss": 4.373330384896676, "tokens_seen": 203882496 }, { "epoch": 0.06, "learning_rate": 0.0007581464393179539, "loss": 1.6905, "theoretical_loss": 4.373156177332536, "tokens_seen": 203948032 }, { "epoch": 0.06, "learning_rate": 0.0007581303911735206, "loss": 1.7408, "theoretical_loss": 4.372982041407161, "tokens_seen": 204013568 }, { "epoch": 0.06, "learning_rate": 0.0007581143430290873, "loss": 1.64, "theoretical_loss": 4.372807977068092, "tokens_seen": 204079104 }, { "epoch": 0.06, "learning_rate": 0.000758098294884654, "loss": 1.6473, "theoretical_loss": 4.372633984262928, "tokens_seen": 204144640 }, { "epoch": 0.06, "learning_rate": 0.0007580822467402207, "loss": 1.6681, "theoretical_loss": 4.372460062939318, "tokens_seen": 204210176 }, { "epoch": 0.06, "learning_rate": 0.0007580661985957874, "loss": 1.6786, "theoretical_loss": 4.372286213044972, "tokens_seen": 204275712 }, { "epoch": 0.06, "learning_rate": 0.0007580501504513541, "loss": 1.7793, "theoretical_loss": 4.37211243452765, "tokens_seen": 204341248 }, { "epoch": 0.06, "learning_rate": 0.0007580341023069208, "loss": 1.5185, "theoretical_loss": 4.37193872733517, "tokens_seen": 204406784 }, { "epoch": 0.06, "learning_rate": 0.0007580180541624875, "loss": 1.7019, "theoretical_loss": 4.371765091415404, "tokens_seen": 204472320 }, { "epoch": 0.06, "learning_rate": 0.0007580020060180542, "loss": 1.7595, "theoretical_loss": 4.371591526716279, "tokens_seen": 204537856 }, { "epoch": 0.06, "learning_rate": 0.0007579859578736209, "loss": 1.7805, "theoretical_loss": 4.371418033185777, "tokens_seen": 204603392 }, { "epoch": 0.06, "learning_rate": 0.0007579699097291876, "loss": 1.5919, "theoretical_loss": 4.3712446107719325, "tokens_seen": 204668928 }, { "epoch": 0.06, "learning_rate": 0.0007579538615847543, "loss": 1.7362, "theoretical_loss": 4.371071259422838, "tokens_seen": 204734464 }, { "epoch": 0.06, "learning_rate": 0.000757937813440321, "loss": 1.6875, "theoretical_loss": 4.370897979086636, "tokens_seen": 204800000 }, { "epoch": 0.06, "learning_rate": 0.0007579217652958877, "loss": 1.8177, "theoretical_loss": 4.370724769711529, "tokens_seen": 204865536 }, { "epoch": 0.06, "learning_rate": 0.0007579057171514544, "loss": 1.7856, "theoretical_loss": 4.370551631245769, "tokens_seen": 204931072 }, { "epoch": 0.06, "learning_rate": 0.0007578896690070211, "loss": 1.6602, "theoretical_loss": 4.370378563637667, "tokens_seen": 204996608 }, { "epoch": 0.06, "learning_rate": 0.0007578736208625878, "loss": 1.7391, "theoretical_loss": 4.3702055668355815, "tokens_seen": 205062144 }, { "epoch": 0.06, "learning_rate": 0.0007578575727181545, "loss": 1.7766, "theoretical_loss": 4.370032640787931, "tokens_seen": 205127680 }, { "epoch": 0.06, "learning_rate": 0.0007578415245737211, "loss": 1.7982, "theoretical_loss": 4.369859785443188, "tokens_seen": 205193216 }, { "epoch": 0.06, "learning_rate": 0.0007578254764292879, "loss": 1.6197, "theoretical_loss": 4.369687000749873, "tokens_seen": 205258752 }, { "epoch": 0.06, "learning_rate": 0.0007578094282848545, "loss": 1.7441, "theoretical_loss": 4.3695142866565675, "tokens_seen": 205324288 }, { "epoch": 0.06, "learning_rate": 0.0007577933801404213, "loss": 1.6629, "theoretical_loss": 4.369341643111902, "tokens_seen": 205389824 }, { "epoch": 0.06, "learning_rate": 0.0007577773319959879, "loss": 1.7171, "theoretical_loss": 4.369169070064563, "tokens_seen": 205455360 }, { "epoch": 0.06, "learning_rate": 0.0007577612838515547, "loss": 1.6443, "theoretical_loss": 4.368996567463292, "tokens_seen": 205520896 }, { "epoch": 0.06, "learning_rate": 0.0007577452357071213, "loss": 1.6624, "theoretical_loss": 4.3688241352568795, "tokens_seen": 205586432 }, { "epoch": 0.06, "learning_rate": 0.0007577291875626881, "loss": 1.8175, "theoretical_loss": 4.368651773394173, "tokens_seen": 205651968 }, { "epoch": 0.06, "learning_rate": 0.0007577131394182547, "loss": 1.5977, "theoretical_loss": 4.368479481824075, "tokens_seen": 205717504 }, { "epoch": 0.06, "learning_rate": 0.0007576970912738215, "loss": 1.6515, "theoretical_loss": 4.368307260495536, "tokens_seen": 205783040 }, { "epoch": 0.06, "learning_rate": 0.0007576810431293881, "loss": 1.6509, "theoretical_loss": 4.368135109357564, "tokens_seen": 205848576 }, { "epoch": 0.06, "learning_rate": 0.0007576649949849549, "loss": 1.7278, "theoretical_loss": 4.36796302835922, "tokens_seen": 205914112 }, { "epoch": 0.06, "learning_rate": 0.0007576489468405215, "loss": 1.5491, "theoretical_loss": 4.367791017449618, "tokens_seen": 205979648 }, { "epoch": 0.06, "learning_rate": 0.0007576328986960883, "loss": 1.555, "theoretical_loss": 4.367619076577923, "tokens_seen": 206045184 }, { "epoch": 0.06, "learning_rate": 0.000757616850551655, "loss": 1.6893, "theoretical_loss": 4.367447205693354, "tokens_seen": 206110720 }, { "epoch": 0.06, "learning_rate": 0.0007576008024072217, "loss": 1.7705, "theoretical_loss": 4.367275404745186, "tokens_seen": 206176256 }, { "epoch": 0.06, "learning_rate": 0.0007575847542627884, "loss": 1.7363, "theoretical_loss": 4.367103673682742, "tokens_seen": 206241792 }, { "epoch": 0.06, "learning_rate": 0.0007575687061183551, "loss": 1.7842, "theoretical_loss": 4.3669320124554, "tokens_seen": 206307328 }, { "epoch": 0.06, "learning_rate": 0.0007575526579739218, "loss": 1.6683, "theoretical_loss": 4.366760421012592, "tokens_seen": 206372864 }, { "epoch": 0.06, "objective/train/docs_used": 75241, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.8373074531555176, "objective/train/theoretical_loss": 4.366588899303801, "objective/train/tokens_used": 226898400, "theoretical_loss": 4.366588899303801, "tokens_seen": 206438400 }, { "epoch": 0.06, "learning_rate": 0.0007575366098294885, "loss": 1.7905, "theoretical_loss": 4.366588899303801, "tokens_seen": 206438400 }, { "epoch": 0.06, "learning_rate": 0.0007575205616850552, "loss": 1.7056, "theoretical_loss": 4.366417447278565, "tokens_seen": 206503936 }, { "epoch": 0.06, "learning_rate": 0.0007575045135406219, "loss": 1.6869, "theoretical_loss": 4.366246064886468, "tokens_seen": 206569472 }, { "epoch": 0.06, "learning_rate": 0.0007574884653961886, "loss": 1.7188, "theoretical_loss": 4.3660747520771555, "tokens_seen": 206635008 }, { "epoch": 0.06, "learning_rate": 0.0007574724172517553, "loss": 1.6279, "theoretical_loss": 4.36590350880032, "tokens_seen": 206700544 }, { "epoch": 0.06, "learning_rate": 0.000757456369107322, "loss": 1.6978, "theoretical_loss": 4.365732335005706, "tokens_seen": 206766080 }, { "epoch": 0.06, "learning_rate": 0.0007574403209628887, "loss": 1.647, "theoretical_loss": 4.365561230643112, "tokens_seen": 206831616 }, { "epoch": 0.06, "learning_rate": 0.0007574242728184554, "loss": 1.691, "theoretical_loss": 4.365390195662389, "tokens_seen": 206897152 }, { "epoch": 0.06, "learning_rate": 0.0007574082246740221, "loss": 1.7532, "theoretical_loss": 4.365219230013438, "tokens_seen": 206962688 }, { "epoch": 0.06, "learning_rate": 0.0007573921765295888, "loss": 1.7129, "theoretical_loss": 4.3650483336462145, "tokens_seen": 207028224 }, { "epoch": 0.06, "learning_rate": 0.0007573761283851555, "loss": 1.7066, "theoretical_loss": 4.364877506510723, "tokens_seen": 207093760 }, { "epoch": 0.06, "learning_rate": 0.0007573600802407222, "loss": 1.6376, "theoretical_loss": 4.3647067485570235, "tokens_seen": 207159296 }, { "epoch": 0.06, "learning_rate": 0.0007573440320962889, "loss": 1.5943, "theoretical_loss": 4.364536059735224, "tokens_seen": 207224832 }, { "epoch": 0.06, "learning_rate": 0.0007573279839518556, "loss": 1.7438, "theoretical_loss": 4.364365439995488, "tokens_seen": 207290368 }, { "epoch": 0.06, "learning_rate": 0.0007573119358074223, "loss": 1.7198, "theoretical_loss": 4.3641948892880285, "tokens_seen": 207355904 }, { "epoch": 0.06, "learning_rate": 0.000757295887662989, "loss": 1.7217, "theoretical_loss": 4.364024407563109, "tokens_seen": 207421440 }, { "epoch": 0.06, "learning_rate": 0.0007572798395185557, "loss": 1.6305, "theoretical_loss": 4.363853994771048, "tokens_seen": 207486976 }, { "epoch": 0.06, "learning_rate": 0.0007572637913741224, "loss": 1.713, "theoretical_loss": 4.363683650862212, "tokens_seen": 207552512 }, { "epoch": 0.06, "learning_rate": 0.0007572477432296891, "loss": 1.6457, "theoretical_loss": 4.36351337578702, "tokens_seen": 207618048 }, { "epoch": 0.06, "learning_rate": 0.0007572316950852558, "loss": 1.6328, "theoretical_loss": 4.363343169495945, "tokens_seen": 207683584 }, { "epoch": 0.06, "learning_rate": 0.0007572156469408225, "loss": 1.717, "theoretical_loss": 4.363173031939506, "tokens_seen": 207749120 }, { "epoch": 0.06, "learning_rate": 0.0007571995987963892, "loss": 1.6263, "theoretical_loss": 4.363002963068277, "tokens_seen": 207814656 }, { "epoch": 0.06, "learning_rate": 0.0007571835506519559, "loss": 1.7193, "theoretical_loss": 4.362832962832883, "tokens_seen": 207880192 }, { "epoch": 0.06, "learning_rate": 0.0007571675025075226, "loss": 1.7143, "theoretical_loss": 4.362663031183999, "tokens_seen": 207945728 }, { "epoch": 0.06, "learning_rate": 0.0007571514543630893, "loss": 1.6003, "theoretical_loss": 4.36249316807235, "tokens_seen": 208011264 }, { "epoch": 0.06, "learning_rate": 0.000757135406218656, "loss": 1.7562, "theoretical_loss": 4.3623233734487155, "tokens_seen": 208076800 }, { "epoch": 0.06, "learning_rate": 0.0007571193580742227, "loss": 1.7425, "theoretical_loss": 4.362153647263921, "tokens_seen": 208142336 }, { "epoch": 0.06, "learning_rate": 0.0007571033099297894, "loss": 1.6691, "theoretical_loss": 4.361983989468847, "tokens_seen": 208207872 }, { "epoch": 0.06, "learning_rate": 0.0007570872617853561, "loss": 1.5959, "theoretical_loss": 4.3618144000144214, "tokens_seen": 208273408 }, { "epoch": 0.06, "learning_rate": 0.0007570712136409228, "loss": 1.6334, "theoretical_loss": 4.361644878851626, "tokens_seen": 208338944 }, { "epoch": 0.06, "learning_rate": 0.0007570551654964895, "loss": 1.832, "theoretical_loss": 4.36147542593149, "tokens_seen": 208404480 }, { "epoch": 0.06, "learning_rate": 0.0007570391173520562, "loss": 1.7518, "theoretical_loss": 4.361306041205095, "tokens_seen": 208470016 }, { "epoch": 0.06, "learning_rate": 0.0007570230692076229, "loss": 1.7394, "theoretical_loss": 4.361136724623573, "tokens_seen": 208535552 }, { "epoch": 0.06, "learning_rate": 0.0007570070210631896, "loss": 1.6212, "theoretical_loss": 4.360967476138105, "tokens_seen": 208601088 }, { "epoch": 0.06, "learning_rate": 0.0007569909729187563, "loss": 1.732, "theoretical_loss": 4.360798295699925, "tokens_seen": 208666624 }, { "epoch": 0.06, "learning_rate": 0.000756974924774323, "loss": 1.7454, "theoretical_loss": 4.360629183260313, "tokens_seen": 208732160 }, { "epoch": 0.06, "learning_rate": 0.0007569588766298897, "loss": 1.6911, "theoretical_loss": 4.360460138770604, "tokens_seen": 208797696 }, { "epoch": 0.06, "learning_rate": 0.0007569428284854564, "loss": 1.5386, "theoretical_loss": 4.360291162182179, "tokens_seen": 208863232 }, { "epoch": 0.06, "learning_rate": 0.0007569267803410231, "loss": 1.6819, "theoretical_loss": 4.360122253446472, "tokens_seen": 208928768 }, { "epoch": 0.06, "learning_rate": 0.0007569107321965898, "loss": 1.7132, "theoretical_loss": 4.359953412514965, "tokens_seen": 208994304 }, { "epoch": 0.06, "learning_rate": 0.0007568946840521565, "loss": 1.6954, "theoretical_loss": 4.359784639339191, "tokens_seen": 209059840 }, { "epoch": 0.06, "learning_rate": 0.0007568786359077232, "loss": 1.5804, "theoretical_loss": 4.359615933870732, "tokens_seen": 209125376 }, { "epoch": 0.06, "learning_rate": 0.0007568625877632899, "loss": 1.826, "theoretical_loss": 4.359447296061219, "tokens_seen": 209190912 }, { "epoch": 0.06, "learning_rate": 0.0007568465396188566, "loss": 1.6614, "theoretical_loss": 4.359278725862336, "tokens_seen": 209256448 }, { "epoch": 0.06, "learning_rate": 0.0007568304914744233, "loss": 1.7834, "theoretical_loss": 4.359110223225814, "tokens_seen": 209321984 }, { "epoch": 0.06, "learning_rate": 0.00075681444332999, "loss": 1.7298, "theoretical_loss": 4.358941788103433, "tokens_seen": 209387520 }, { "epoch": 0.06, "learning_rate": 0.0007567983951855567, "loss": 1.6115, "theoretical_loss": 4.358773420447024, "tokens_seen": 209453056 }, { "epoch": 0.06, "learning_rate": 0.0007567823470411234, "loss": 1.7582, "theoretical_loss": 4.358605120208469, "tokens_seen": 209518592 }, { "epoch": 0.06, "learning_rate": 0.0007567662988966901, "loss": 1.6202, "theoretical_loss": 4.358436887339694, "tokens_seen": 209584128 }, { "epoch": 0.06, "learning_rate": 0.0007567502507522569, "loss": 1.6705, "theoretical_loss": 4.35826872179268, "tokens_seen": 209649664 }, { "epoch": 0.06, "objective/train/docs_used": 76142, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.782550573348999, "objective/train/theoretical_loss": 4.358100623519453, "objective/train/tokens_used": 230175200, "theoretical_loss": 4.358100623519453, "tokens_seen": 209715200 }, { "epoch": 0.06, "learning_rate": 0.0007567342026078235, "loss": 1.6927, "theoretical_loss": 4.358100623519453, "tokens_seen": 209715200 }, { "epoch": 0.06, "learning_rate": 0.0007567181544633903, "loss": 1.6178, "theoretical_loss": 4.357932592472092, "tokens_seen": 209780736 }, { "epoch": 0.06, "learning_rate": 0.0007567021063189569, "loss": 1.6931, "theoretical_loss": 4.357764628602721, "tokens_seen": 209846272 }, { "epoch": 0.06, "learning_rate": 0.0007566860581745237, "loss": 1.6771, "theoretical_loss": 4.357596731863517, "tokens_seen": 209911808 }, { "epoch": 0.06, "learning_rate": 0.0007566700100300903, "loss": 1.7497, "theoretical_loss": 4.357428902206704, "tokens_seen": 209977344 }, { "epoch": 0.06, "learning_rate": 0.0007566539618856571, "loss": 1.792, "theoretical_loss": 4.357261139584553, "tokens_seen": 210042880 }, { "epoch": 0.06, "learning_rate": 0.0007566379137412237, "loss": 1.7182, "theoretical_loss": 4.357093443949388, "tokens_seen": 210108416 }, { "epoch": 0.06, "learning_rate": 0.0007566218655967905, "loss": 1.7378, "theoretical_loss": 4.356925815253579, "tokens_seen": 210173952 }, { "epoch": 0.06, "learning_rate": 0.0007566058174523571, "loss": 1.623, "theoretical_loss": 4.356758253449544, "tokens_seen": 210239488 }, { "epoch": 0.06, "learning_rate": 0.0007565897693079239, "loss": 1.6864, "theoretical_loss": 4.356590758489753, "tokens_seen": 210305024 }, { "epoch": 0.06, "learning_rate": 0.0007565737211634905, "loss": 1.6433, "theoretical_loss": 4.356423330326722, "tokens_seen": 210370560 }, { "epoch": 0.06, "learning_rate": 0.0007565576730190572, "loss": 1.6874, "theoretical_loss": 4.356255968913015, "tokens_seen": 210436096 }, { "epoch": 0.06, "learning_rate": 0.000756541624874624, "loss": 1.6989, "theoretical_loss": 4.356088674201246, "tokens_seen": 210501632 }, { "epoch": 0.06, "learning_rate": 0.0007565255767301906, "loss": 1.6619, "theoretical_loss": 4.355921446144077, "tokens_seen": 210567168 }, { "epoch": 0.06, "learning_rate": 0.0007565095285857573, "loss": 1.6612, "theoretical_loss": 4.355754284694218, "tokens_seen": 210632704 }, { "epoch": 0.06, "learning_rate": 0.000756493480441324, "loss": 1.8118, "theoretical_loss": 4.355587189804427, "tokens_seen": 210698240 }, { "epoch": 0.06, "learning_rate": 0.0007564774322968907, "loss": 1.6183, "theoretical_loss": 4.355420161427512, "tokens_seen": 210763776 }, { "epoch": 0.06, "learning_rate": 0.0007564613841524574, "loss": 1.6552, "theoretical_loss": 4.355253199516326, "tokens_seen": 210829312 }, { "epoch": 0.06, "learning_rate": 0.0007564453360080242, "loss": 1.6956, "theoretical_loss": 4.355086304023773, "tokens_seen": 210894848 }, { "epoch": 0.06, "learning_rate": 0.0007564292878635908, "loss": 1.6164, "theoretical_loss": 4.354919474902804, "tokens_seen": 210960384 }, { "epoch": 0.06, "learning_rate": 0.0007564132397191574, "loss": 1.6407, "theoretical_loss": 4.354752712106417, "tokens_seen": 211025920 }, { "epoch": 0.06, "learning_rate": 0.0007563971915747242, "loss": 1.6663, "theoretical_loss": 4.354586015587659, "tokens_seen": 211091456 }, { "epoch": 0.06, "learning_rate": 0.0007563811434302908, "loss": 1.7245, "theoretical_loss": 4.354419385299623, "tokens_seen": 211156992 }, { "epoch": 0.06, "learning_rate": 0.0007563650952858576, "loss": 1.6848, "theoretical_loss": 4.354252821195453, "tokens_seen": 211222528 }, { "epoch": 0.06, "learning_rate": 0.0007563490471414242, "loss": 1.7278, "theoretical_loss": 4.354086323228338, "tokens_seen": 211288064 }, { "epoch": 0.06, "learning_rate": 0.000756332998996991, "loss": 1.7467, "theoretical_loss": 4.353919891351516, "tokens_seen": 211353600 }, { "epoch": 0.06, "learning_rate": 0.0007563169508525576, "loss": 1.6958, "theoretical_loss": 4.35375352551827, "tokens_seen": 211419136 }, { "epoch": 0.06, "learning_rate": 0.0007563009027081244, "loss": 1.7661, "theoretical_loss": 4.353587225681935, "tokens_seen": 211484672 }, { "epoch": 0.06, "learning_rate": 0.000756284854563691, "loss": 1.7008, "theoretical_loss": 4.35342099179589, "tokens_seen": 211550208 }, { "epoch": 0.06, "learning_rate": 0.0007562688064192578, "loss": 1.6198, "theoretical_loss": 4.353254823813561, "tokens_seen": 211615744 }, { "epoch": 0.06, "learning_rate": 0.0007562527582748245, "loss": 1.6464, "theoretical_loss": 4.353088721688424, "tokens_seen": 211681280 }, { "epoch": 0.06, "learning_rate": 0.0007562367101303912, "loss": 1.6186, "theoretical_loss": 4.352922685374001, "tokens_seen": 211746816 }, { "epoch": 0.06, "learning_rate": 0.0007562206619859579, "loss": 1.7021, "theoretical_loss": 4.352756714823859, "tokens_seen": 211812352 }, { "epoch": 0.06, "learning_rate": 0.0007562046138415246, "loss": 1.6447, "theoretical_loss": 4.352590809991616, "tokens_seen": 211877888 }, { "epoch": 0.06, "learning_rate": 0.0007561885656970913, "loss": 1.7092, "theoretical_loss": 4.352424970830935, "tokens_seen": 211943424 }, { "epoch": 0.06, "learning_rate": 0.000756172517552658, "loss": 1.6352, "theoretical_loss": 4.352259197295525, "tokens_seen": 212008960 }, { "epoch": 0.06, "learning_rate": 0.0007561564694082247, "loss": 1.6442, "theoretical_loss": 4.352093489339143, "tokens_seen": 212074496 }, { "epoch": 0.06, "learning_rate": 0.0007561404212637914, "loss": 1.6247, "theoretical_loss": 4.351927846915595, "tokens_seen": 212140032 }, { "epoch": 0.06, "learning_rate": 0.0007561243731193581, "loss": 1.7138, "theoretical_loss": 4.35176226997873, "tokens_seen": 212205568 }, { "epoch": 0.06, "learning_rate": 0.0007561083249749248, "loss": 1.7305, "theoretical_loss": 4.351596758482445, "tokens_seen": 212271104 }, { "epoch": 0.06, "learning_rate": 0.0007560922768304915, "loss": 1.6121, "theoretical_loss": 4.351431312380684, "tokens_seen": 212336640 }, { "epoch": 0.06, "learning_rate": 0.0007560762286860582, "loss": 1.6804, "theoretical_loss": 4.351265931627439, "tokens_seen": 212402176 }, { "epoch": 0.06, "learning_rate": 0.0007560601805416249, "loss": 1.664, "theoretical_loss": 4.351100616176747, "tokens_seen": 212467712 }, { "epoch": 0.06, "learning_rate": 0.0007560441323971916, "loss": 1.6836, "theoretical_loss": 4.350935365982691, "tokens_seen": 212533248 }, { "epoch": 0.06, "learning_rate": 0.0007560280842527583, "loss": 1.7276, "theoretical_loss": 4.350770180999402, "tokens_seen": 212598784 }, { "epoch": 0.06, "learning_rate": 0.000756012036108325, "loss": 1.6969, "theoretical_loss": 4.3506050611810565, "tokens_seen": 212664320 }, { "epoch": 0.06, "learning_rate": 0.0007559959879638917, "loss": 1.5556, "theoretical_loss": 4.350440006481877, "tokens_seen": 212729856 }, { "epoch": 0.06, "learning_rate": 0.0007559799398194584, "loss": 1.7229, "theoretical_loss": 4.350275016856132, "tokens_seen": 212795392 }, { "epoch": 0.06, "learning_rate": 0.0007559638916750251, "loss": 1.6752, "theoretical_loss": 4.350110092258138, "tokens_seen": 212860928 }, { "epoch": 0.06, "learning_rate": 0.0007559478435305918, "loss": 1.6807, "theoretical_loss": 4.349945232642256, "tokens_seen": 212926464 }, { "epoch": 0.06, "objective/train/docs_used": 77362, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6674554347991943, "objective/train/theoretical_loss": 4.3497804379628935, "objective/train/tokens_used": 233452000, "theoretical_loss": 4.3497804379628935, "tokens_seen": 212992000 }, { "epoch": 0.06, "learning_rate": 0.0007559317953861585, "loss": 1.7286, "theoretical_loss": 4.3497804379628935, "tokens_seen": 212992000 }, { "epoch": 0.06, "learning_rate": 0.0007559157472417252, "loss": 1.6309, "theoretical_loss": 4.349615708174504, "tokens_seen": 213057536 }, { "epoch": 0.06, "learning_rate": 0.0007558996990972919, "loss": 1.589, "theoretical_loss": 4.3494510432315865, "tokens_seen": 213123072 }, { "epoch": 0.06, "learning_rate": 0.0007558836509528586, "loss": 1.6323, "theoretical_loss": 4.349286443088687, "tokens_seen": 213188608 }, { "epoch": 0.06, "learning_rate": 0.0007558676028084253, "loss": 1.6095, "theoretical_loss": 4.349121907700396, "tokens_seen": 213254144 }, { "epoch": 0.06, "learning_rate": 0.000755851554663992, "loss": 1.7698, "theoretical_loss": 4.348957437021351, "tokens_seen": 213319680 }, { "epoch": 0.06, "learning_rate": 0.0007558355065195587, "loss": 1.7637, "theoretical_loss": 4.348793031006235, "tokens_seen": 213385216 }, { "epoch": 0.06, "learning_rate": 0.0007558194583751254, "loss": 1.636, "theoretical_loss": 4.348628689609775, "tokens_seen": 213450752 }, { "epoch": 0.06, "learning_rate": 0.0007558034102306921, "loss": 1.6531, "theoretical_loss": 4.348464412786746, "tokens_seen": 213516288 }, { "epoch": 0.06, "learning_rate": 0.0007557873620862588, "loss": 1.7114, "theoretical_loss": 4.348300200491966, "tokens_seen": 213581824 }, { "epoch": 0.06, "learning_rate": 0.0007557713139418255, "loss": 1.6954, "theoretical_loss": 4.348136052680301, "tokens_seen": 213647360 }, { "epoch": 0.06, "learning_rate": 0.0007557552657973922, "loss": 1.7568, "theoretical_loss": 4.347971969306662, "tokens_seen": 213712896 }, { "epoch": 0.06, "learning_rate": 0.0007557392176529589, "loss": 1.5466, "theoretical_loss": 4.347807950326002, "tokens_seen": 213778432 }, { "epoch": 0.06, "learning_rate": 0.0007557231695085256, "loss": 1.7092, "theoretical_loss": 4.347643995693325, "tokens_seen": 213843968 }, { "epoch": 0.06, "learning_rate": 0.0007557071213640923, "loss": 1.6906, "theoretical_loss": 4.347480105363674, "tokens_seen": 213909504 }, { "epoch": 0.06, "learning_rate": 0.000755691073219659, "loss": 1.767, "theoretical_loss": 4.3473162792921425, "tokens_seen": 213975040 }, { "epoch": 0.06, "learning_rate": 0.0007556750250752257, "loss": 1.5911, "theoretical_loss": 4.347152517433866, "tokens_seen": 214040576 }, { "epoch": 0.06, "learning_rate": 0.0007556589769307924, "loss": 1.5156, "theoretical_loss": 4.346988819744026, "tokens_seen": 214106112 }, { "epoch": 0.06, "learning_rate": 0.0007556429287863591, "loss": 1.6893, "theoretical_loss": 4.346825186177848, "tokens_seen": 214171648 }, { "epoch": 0.06, "learning_rate": 0.0007556268806419258, "loss": 1.6864, "theoretical_loss": 4.346661616690605, "tokens_seen": 214237184 }, { "epoch": 0.06, "learning_rate": 0.0007556108324974925, "loss": 1.7354, "theoretical_loss": 4.346498111237612, "tokens_seen": 214302720 }, { "epoch": 0.06, "learning_rate": 0.0007555947843530592, "loss": 1.6528, "theoretical_loss": 4.346334669774231, "tokens_seen": 214368256 }, { "epoch": 0.06, "learning_rate": 0.0007555787362086259, "loss": 1.5685, "theoretical_loss": 4.346171292255867, "tokens_seen": 214433792 }, { "epoch": 0.06, "learning_rate": 0.0007555626880641926, "loss": 1.6147, "theoretical_loss": 4.34600797863797, "tokens_seen": 214499328 }, { "epoch": 0.07, "learning_rate": 0.0007555466399197594, "loss": 1.6934, "theoretical_loss": 4.345844728876036, "tokens_seen": 214564864 }, { "epoch": 0.07, "learning_rate": 0.000755530591775326, "loss": 1.7624, "theoretical_loss": 4.345681542925604, "tokens_seen": 214630400 }, { "epoch": 0.07, "learning_rate": 0.0007555145436308928, "loss": 1.5814, "theoretical_loss": 4.345518420742259, "tokens_seen": 214695936 }, { "epoch": 0.07, "learning_rate": 0.0007554984954864594, "loss": 1.6765, "theoretical_loss": 4.345355362281628, "tokens_seen": 214761472 }, { "epoch": 0.07, "learning_rate": 0.0007554824473420262, "loss": 1.6003, "theoretical_loss": 4.345192367499386, "tokens_seen": 214827008 }, { "epoch": 0.07, "learning_rate": 0.0007554663991975928, "loss": 1.7416, "theoretical_loss": 4.3450294363512505, "tokens_seen": 214892544 }, { "epoch": 0.07, "learning_rate": 0.0007554503510531596, "loss": 1.7229, "theoretical_loss": 4.34486656879298, "tokens_seen": 214958080 }, { "epoch": 0.07, "learning_rate": 0.0007554343029087262, "loss": 1.557, "theoretical_loss": 4.344703764780386, "tokens_seen": 215023616 }, { "epoch": 0.07, "learning_rate": 0.000755418254764293, "loss": 1.7031, "theoretical_loss": 4.344541024269312, "tokens_seen": 215089152 }, { "epoch": 0.07, "learning_rate": 0.0007554022066198596, "loss": 1.6627, "theoretical_loss": 4.344378347215657, "tokens_seen": 215154688 }, { "epoch": 0.07, "learning_rate": 0.0007553861584754264, "loss": 1.6637, "theoretical_loss": 4.344215733575357, "tokens_seen": 215220224 }, { "epoch": 0.07, "learning_rate": 0.000755370110330993, "loss": 1.5809, "theoretical_loss": 4.344053183304396, "tokens_seen": 215285760 }, { "epoch": 0.07, "learning_rate": 0.0007553540621865598, "loss": 1.7869, "theoretical_loss": 4.343890696358798, "tokens_seen": 215351296 }, { "epoch": 0.07, "learning_rate": 0.0007553380140421264, "loss": 1.4857, "theoretical_loss": 4.343728272694635, "tokens_seen": 215416832 }, { "epoch": 0.07, "learning_rate": 0.0007553219658976932, "loss": 1.4274, "theoretical_loss": 4.34356591226802, "tokens_seen": 215482368 }, { "epoch": 0.07, "learning_rate": 0.0007553059177532598, "loss": 1.7362, "theoretical_loss": 4.343403615035111, "tokens_seen": 215547904 }, { "epoch": 0.07, "learning_rate": 0.0007552898696088266, "loss": 1.6923, "theoretical_loss": 4.343241380952109, "tokens_seen": 215613440 }, { "epoch": 0.07, "learning_rate": 0.0007552738214643932, "loss": 1.7153, "theoretical_loss": 4.34307920997526, "tokens_seen": 215678976 }, { "epoch": 0.07, "learning_rate": 0.00075525777331996, "loss": 1.7254, "theoretical_loss": 4.34291710206085, "tokens_seen": 215744512 }, { "epoch": 0.07, "learning_rate": 0.0007552417251755266, "loss": 1.769, "theoretical_loss": 4.342755057165214, "tokens_seen": 215810048 }, { "epoch": 0.07, "learning_rate": 0.0007552256770310934, "loss": 1.6395, "theoretical_loss": 4.342593075244728, "tokens_seen": 215875584 }, { "epoch": 0.07, "learning_rate": 0.00075520962888666, "loss": 1.6217, "theoretical_loss": 4.342431156255809, "tokens_seen": 215941120 }, { "epoch": 0.07, "learning_rate": 0.0007551935807422268, "loss": 1.5778, "theoretical_loss": 4.3422693001549195, "tokens_seen": 216006656 }, { "epoch": 0.07, "learning_rate": 0.0007551775325977934, "loss": 1.654, "theoretical_loss": 4.342107506898567, "tokens_seen": 216072192 }, { "epoch": 0.07, "learning_rate": 0.0007551614844533601, "loss": 1.6215, "theoretical_loss": 4.3419457764433, "tokens_seen": 216137728 }, { "epoch": 0.07, "learning_rate": 0.0007551454363089268, "loss": 1.5418, "theoretical_loss": 4.34178410874571, "tokens_seen": 216203264 }, { "epoch": 0.07, "objective/train/docs_used": 78526, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.7171001434326172, "objective/train/theoretical_loss": 4.341622503762434, "objective/train/tokens_used": 236728800, "theoretical_loss": 4.341622503762434, "tokens_seen": 216268800 }, { "epoch": 0.07, "learning_rate": 0.0007551293881644935, "loss": 1.6569, "theoretical_loss": 4.341622503762434, "tokens_seen": 216268800 }, { "epoch": 0.07, "learning_rate": 0.0007551133400200602, "loss": 1.6461, "theoretical_loss": 4.341460961450148, "tokens_seen": 216334336 }, { "epoch": 0.07, "learning_rate": 0.0007550972918756269, "loss": 1.5453, "theoretical_loss": 4.341299481765575, "tokens_seen": 216399872 }, { "epoch": 0.07, "learning_rate": 0.0007550812437311937, "loss": 1.6849, "theoretical_loss": 4.34113806466548, "tokens_seen": 216465408 }, { "epoch": 0.07, "learning_rate": 0.0007550651955867603, "loss": 1.7779, "theoretical_loss": 4.340976710106671, "tokens_seen": 216530944 }, { "epoch": 0.07, "learning_rate": 0.000755049147442327, "loss": 1.6228, "theoretical_loss": 4.340815418045995, "tokens_seen": 216596480 }, { "epoch": 0.07, "learning_rate": 0.0007550330992978937, "loss": 1.7109, "theoretical_loss": 4.340654188440349, "tokens_seen": 216662016 }, { "epoch": 0.07, "learning_rate": 0.0007550170511534603, "loss": 1.6088, "theoretical_loss": 4.340493021246668, "tokens_seen": 216727552 }, { "epoch": 0.07, "learning_rate": 0.0007550010030090271, "loss": 1.63, "theoretical_loss": 4.3403319164219285, "tokens_seen": 216793088 }, { "epoch": 0.07, "learning_rate": 0.0007549849548645937, "loss": 1.7283, "theoretical_loss": 4.340170873923155, "tokens_seen": 216858624 }, { "epoch": 0.07, "learning_rate": 0.0007549689067201605, "loss": 1.6471, "theoretical_loss": 4.340009893707408, "tokens_seen": 216924160 }, { "epoch": 0.07, "learning_rate": 0.0007549528585757271, "loss": 1.6254, "theoretical_loss": 4.339848975731796, "tokens_seen": 216989696 }, { "epoch": 0.07, "learning_rate": 0.0007549368104312939, "loss": 1.6024, "theoretical_loss": 4.3396881199534665, "tokens_seen": 217055232 }, { "epoch": 0.07, "learning_rate": 0.0007549207622868605, "loss": 1.6612, "theoretical_loss": 4.339527326329613, "tokens_seen": 217120768 }, { "epoch": 0.07, "learning_rate": 0.0007549047141424273, "loss": 1.6202, "theoretical_loss": 4.3393665948174664, "tokens_seen": 217186304 }, { "epoch": 0.07, "learning_rate": 0.000754888665997994, "loss": 1.6312, "theoretical_loss": 4.339205925374305, "tokens_seen": 217251840 }, { "epoch": 0.07, "learning_rate": 0.0007548726178535607, "loss": 1.5936, "theoretical_loss": 4.339045317957446, "tokens_seen": 217317376 }, { "epoch": 0.07, "learning_rate": 0.0007548565697091274, "loss": 1.6884, "theoretical_loss": 4.33888477252425, "tokens_seen": 217382912 }, { "epoch": 0.07, "learning_rate": 0.0007548405215646941, "loss": 1.6815, "theoretical_loss": 4.33872428903212, "tokens_seen": 217448448 }, { "epoch": 0.07, "learning_rate": 0.0007548244734202608, "loss": 1.6695, "theoretical_loss": 4.338563867438499, "tokens_seen": 217513984 }, { "epoch": 0.07, "learning_rate": 0.0007548084252758275, "loss": 1.6838, "theoretical_loss": 4.338403507700877, "tokens_seen": 217579520 }, { "epoch": 0.07, "learning_rate": 0.0007547923771313942, "loss": 1.4434, "theoretical_loss": 4.33824320977678, "tokens_seen": 217645056 }, { "epoch": 0.07, "learning_rate": 0.0007547763289869609, "loss": 1.6037, "theoretical_loss": 4.33808297362378, "tokens_seen": 217710592 }, { "epoch": 0.07, "learning_rate": 0.0007547602808425276, "loss": 1.6591, "theoretical_loss": 4.337922799199489, "tokens_seen": 217776128 }, { "epoch": 0.07, "learning_rate": 0.0007547442326980943, "loss": 1.6701, "theoretical_loss": 4.337762686461561, "tokens_seen": 217841664 }, { "epoch": 0.07, "learning_rate": 0.000754728184553661, "loss": 1.6801, "theoretical_loss": 4.337602635367694, "tokens_seen": 217907200 }, { "epoch": 0.07, "learning_rate": 0.0007547121364092277, "loss": 1.642, "theoretical_loss": 4.337442645875624, "tokens_seen": 217972736 }, { "epoch": 0.07, "learning_rate": 0.0007546960882647944, "loss": 1.695, "theoretical_loss": 4.337282717943132, "tokens_seen": 218038272 }, { "epoch": 0.07, "learning_rate": 0.0007546800401203611, "loss": 1.7297, "theoretical_loss": 4.337122851528037, "tokens_seen": 218103808 }, { "epoch": 0.07, "learning_rate": 0.0007546639919759278, "loss": 1.5796, "theoretical_loss": 4.336963046588204, "tokens_seen": 218169344 }, { "epoch": 0.07, "learning_rate": 0.0007546479438314945, "loss": 1.6636, "theoretical_loss": 4.336803303081538, "tokens_seen": 218234880 }, { "epoch": 0.07, "learning_rate": 0.0007546318956870612, "loss": 1.608, "theoretical_loss": 4.336643620965982, "tokens_seen": 218300416 }, { "epoch": 0.07, "learning_rate": 0.0007546158475426279, "loss": 1.6229, "theoretical_loss": 4.336484000199524, "tokens_seen": 218365952 }, { "epoch": 0.07, "learning_rate": 0.0007545997993981946, "loss": 1.5969, "theoretical_loss": 4.336324440740195, "tokens_seen": 218431488 }, { "epoch": 0.07, "learning_rate": 0.0007545837512537613, "loss": 1.6826, "theoretical_loss": 4.336164942546061, "tokens_seen": 218497024 }, { "epoch": 0.07, "learning_rate": 0.000754567703109328, "loss": 1.5707, "theoretical_loss": 4.336005505575235, "tokens_seen": 218562560 }, { "epoch": 0.07, "learning_rate": 0.0007545516549648947, "loss": 1.7505, "theoretical_loss": 4.335846129785869, "tokens_seen": 218628096 }, { "epoch": 0.07, "learning_rate": 0.0007545356068204614, "loss": 1.554, "theoretical_loss": 4.335686815136157, "tokens_seen": 218693632 }, { "epoch": 0.07, "learning_rate": 0.0007545195586760281, "loss": 1.6179, "theoretical_loss": 4.335527561584333, "tokens_seen": 218759168 }, { "epoch": 0.07, "learning_rate": 0.0007545035105315948, "loss": 1.6906, "theoretical_loss": 4.3353683690886715, "tokens_seen": 218824704 }, { "epoch": 0.07, "learning_rate": 0.0007544874623871615, "loss": 1.7096, "theoretical_loss": 4.33520923760749, "tokens_seen": 218890240 }, { "epoch": 0.07, "learning_rate": 0.0007544714142427282, "loss": 1.6438, "theoretical_loss": 4.335050167099145, "tokens_seen": 218955776 }, { "epoch": 0.07, "learning_rate": 0.0007544553660982949, "loss": 1.7231, "theoretical_loss": 4.334891157522035, "tokens_seen": 219021312 }, { "epoch": 0.07, "learning_rate": 0.0007544393179538616, "loss": 1.6563, "theoretical_loss": 4.3347322088346, "tokens_seen": 219086848 }, { "epoch": 0.07, "learning_rate": 0.0007544232698094283, "loss": 1.5714, "theoretical_loss": 4.33457332099532, "tokens_seen": 219152384 }, { "epoch": 0.07, "learning_rate": 0.000754407221664995, "loss": 1.5984, "theoretical_loss": 4.334414493962714, "tokens_seen": 219217920 }, { "epoch": 0.07, "learning_rate": 0.0007543911735205617, "loss": 1.6528, "theoretical_loss": 4.334255727695344, "tokens_seen": 219283456 }, { "epoch": 0.07, "learning_rate": 0.0007543751253761284, "loss": 1.6906, "theoretical_loss": 4.334097022151812, "tokens_seen": 219348992 }, { "epoch": 0.07, "learning_rate": 0.0007543590772316951, "loss": 1.71, "theoretical_loss": 4.33393837729076, "tokens_seen": 219414528 }, { "epoch": 0.07, "learning_rate": 0.0007543430290872619, "loss": 1.7512, "theoretical_loss": 4.333779793070871, "tokens_seen": 219480064 }, { "epoch": 0.07, "objective/train/docs_used": 79445, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6968114376068115, "objective/train/theoretical_loss": 4.33362126945087, "objective/train/tokens_used": 240005600, "theoretical_loss": 4.33362126945087, "tokens_seen": 219545600 }, { "epoch": 0.07, "learning_rate": 0.0007543269809428285, "loss": 1.7406, "theoretical_loss": 4.33362126945087, "tokens_seen": 219545600 }, { "epoch": 0.07, "learning_rate": 0.0007543109327983953, "loss": 1.6159, "theoretical_loss": 4.333462806389517, "tokens_seen": 219611136 }, { "epoch": 0.07, "learning_rate": 0.0007542948846539619, "loss": 1.6682, "theoretical_loss": 4.33330440384562, "tokens_seen": 219676672 }, { "epoch": 0.07, "learning_rate": 0.0007542788365095287, "loss": 1.6814, "theoretical_loss": 4.333146061778021, "tokens_seen": 219742208 }, { "epoch": 0.07, "learning_rate": 0.0007542627883650953, "loss": 1.6632, "theoretical_loss": 4.332987780145606, "tokens_seen": 219807744 }, { "epoch": 0.07, "learning_rate": 0.0007542467402206621, "loss": 1.5992, "theoretical_loss": 4.3328295589073, "tokens_seen": 219873280 }, { "epoch": 0.07, "learning_rate": 0.0007542306920762287, "loss": 1.6456, "theoretical_loss": 4.332671398022067, "tokens_seen": 219938816 }, { "epoch": 0.07, "learning_rate": 0.0007542146439317955, "loss": 1.632, "theoretical_loss": 4.332513297448912, "tokens_seen": 220004352 }, { "epoch": 0.07, "learning_rate": 0.0007541985957873621, "loss": 1.7385, "theoretical_loss": 4.332355257146881, "tokens_seen": 220069888 }, { "epoch": 0.07, "learning_rate": 0.0007541825476429289, "loss": 1.5406, "theoretical_loss": 4.3321972770750605, "tokens_seen": 220135424 }, { "epoch": 0.07, "learning_rate": 0.0007541664994984955, "loss": 1.6117, "theoretical_loss": 4.332039357192572, "tokens_seen": 220200960 }, { "epoch": 0.07, "learning_rate": 0.0007541504513540623, "loss": 1.6715, "theoretical_loss": 4.331881497458584, "tokens_seen": 220266496 }, { "epoch": 0.07, "learning_rate": 0.0007541344032096289, "loss": 1.6171, "theoretical_loss": 4.3317236978323, "tokens_seen": 220332032 }, { "epoch": 0.07, "learning_rate": 0.0007541183550651957, "loss": 1.6819, "theoretical_loss": 4.3315659582729635, "tokens_seen": 220397568 }, { "epoch": 0.07, "learning_rate": 0.0007541023069207623, "loss": 1.5839, "theoretical_loss": 4.331408278739861, "tokens_seen": 220463104 }, { "epoch": 0.07, "learning_rate": 0.0007540862587763291, "loss": 1.6511, "theoretical_loss": 4.331250659192316, "tokens_seen": 220528640 }, { "epoch": 0.07, "learning_rate": 0.0007540702106318957, "loss": 1.7196, "theoretical_loss": 4.33109309958969, "tokens_seen": 220594176 }, { "epoch": 0.07, "learning_rate": 0.0007540541624874625, "loss": 1.6633, "theoretical_loss": 4.330935599891389, "tokens_seen": 220659712 }, { "epoch": 0.07, "learning_rate": 0.0007540381143430291, "loss": 1.6971, "theoretical_loss": 4.330778160056855, "tokens_seen": 220725248 }, { "epoch": 0.07, "learning_rate": 0.0007540220661985959, "loss": 1.7276, "theoretical_loss": 4.330620780045571, "tokens_seen": 220790784 }, { "epoch": 0.07, "learning_rate": 0.0007540060180541625, "loss": 1.7717, "theoretical_loss": 4.330463459817057, "tokens_seen": 220856320 }, { "epoch": 0.07, "learning_rate": 0.0007539899699097293, "loss": 1.6917, "theoretical_loss": 4.3303061993308765, "tokens_seen": 220921856 }, { "epoch": 0.07, "learning_rate": 0.0007539739217652959, "loss": 1.6614, "theoretical_loss": 4.330148998546628, "tokens_seen": 220987392 }, { "epoch": 0.07, "learning_rate": 0.0007539578736208627, "loss": 1.6307, "theoretical_loss": 4.329991857423953, "tokens_seen": 221052928 }, { "epoch": 0.07, "learning_rate": 0.0007539418254764293, "loss": 1.5909, "theoretical_loss": 4.329834775922529, "tokens_seen": 221118464 }, { "epoch": 0.07, "learning_rate": 0.0007539257773319961, "loss": 1.7151, "theoretical_loss": 4.329677754002074, "tokens_seen": 221184000 }, { "epoch": 0.07, "learning_rate": 0.0007539097291875627, "loss": 1.6996, "theoretical_loss": 4.329520791622348, "tokens_seen": 221249536 }, { "epoch": 0.07, "learning_rate": 0.0007538936810431295, "loss": 1.6523, "theoretical_loss": 4.329363888743145, "tokens_seen": 221315072 }, { "epoch": 0.07, "learning_rate": 0.0007538776328986961, "loss": 1.6329, "theoretical_loss": 4.329207045324301, "tokens_seen": 221380608 }, { "epoch": 0.07, "learning_rate": 0.0007538615847542629, "loss": 1.6521, "theoretical_loss": 4.3290502613256905, "tokens_seen": 221446144 }, { "epoch": 0.07, "learning_rate": 0.0007538455366098295, "loss": 1.8236, "theoretical_loss": 4.328893536707228, "tokens_seen": 221511680 }, { "epoch": 0.07, "learning_rate": 0.0007538294884653963, "loss": 1.7811, "theoretical_loss": 4.328736871428864, "tokens_seen": 221577216 }, { "epoch": 0.07, "learning_rate": 0.000753813440320963, "loss": 1.6121, "theoretical_loss": 4.328580265450592, "tokens_seen": 221642752 }, { "epoch": 0.07, "learning_rate": 0.0007537973921765297, "loss": 1.6411, "theoretical_loss": 4.32842371873244, "tokens_seen": 221708288 }, { "epoch": 0.07, "learning_rate": 0.0007537813440320963, "loss": 1.7128, "theoretical_loss": 4.328267231234476, "tokens_seen": 221773824 }, { "epoch": 0.07, "learning_rate": 0.000753765295887663, "loss": 1.7062, "theoretical_loss": 4.328110802916811, "tokens_seen": 221839360 }, { "epoch": 0.07, "learning_rate": 0.0007537492477432297, "loss": 1.8148, "theoretical_loss": 4.327954433739588, "tokens_seen": 221904896 }, { "epoch": 0.07, "learning_rate": 0.0007537331995987964, "loss": 1.6588, "theoretical_loss": 4.327798123662992, "tokens_seen": 221970432 }, { "epoch": 0.07, "learning_rate": 0.0007537171514543632, "loss": 1.5768, "theoretical_loss": 4.327641872647248, "tokens_seen": 222035968 }, { "epoch": 0.07, "learning_rate": 0.0007537011033099298, "loss": 1.7152, "theoretical_loss": 4.327485680652615, "tokens_seen": 222101504 }, { "epoch": 0.07, "learning_rate": 0.0007536850551654966, "loss": 1.6536, "theoretical_loss": 4.327329547639396, "tokens_seen": 222167040 }, { "epoch": 0.07, "learning_rate": 0.0007536690070210632, "loss": 1.6875, "theoretical_loss": 4.327173473567927, "tokens_seen": 222232576 }, { "epoch": 0.07, "learning_rate": 0.00075365295887663, "loss": 1.7102, "theoretical_loss": 4.327017458398587, "tokens_seen": 222298112 }, { "epoch": 0.07, "learning_rate": 0.0007536369107321966, "loss": 1.6412, "theoretical_loss": 4.32686150209179, "tokens_seen": 222363648 }, { "epoch": 0.07, "learning_rate": 0.0007536208625877632, "loss": 1.6459, "theoretical_loss": 4.32670560460799, "tokens_seen": 222429184 }, { "epoch": 0.07, "learning_rate": 0.00075360481444333, "loss": 1.5749, "theoretical_loss": 4.326549765907679, "tokens_seen": 222494720 }, { "epoch": 0.07, "learning_rate": 0.0007535887662988966, "loss": 1.7364, "theoretical_loss": 4.326393985951386, "tokens_seen": 222560256 }, { "epoch": 0.07, "learning_rate": 0.0007535727181544634, "loss": 1.5982, "theoretical_loss": 4.326238264699679, "tokens_seen": 222625792 }, { "epoch": 0.07, "learning_rate": 0.00075355667001003, "loss": 1.7705, "theoretical_loss": 4.326082602113164, "tokens_seen": 222691328 }, { "epoch": 0.07, "learning_rate": 0.0007535406218655968, "loss": 1.6681, "theoretical_loss": 4.325926998152487, "tokens_seen": 222756864 }, { "epoch": 0.07, "objective/train/docs_used": 80592, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5793734788894653, "objective/train/theoretical_loss": 4.325771452778328, "objective/train/tokens_used": 243282400, "theoretical_loss": 4.325771452778328, "tokens_seen": 222822400 }, { "epoch": 0.07, "learning_rate": 0.0007535245737211634, "loss": 1.6079, "theoretical_loss": 4.325771452778328, "tokens_seen": 222822400 }, { "epoch": 0.07, "learning_rate": 0.0007535085255767302, "loss": 1.731, "theoretical_loss": 4.325615965951408, "tokens_seen": 222887936 }, { "epoch": 0.07, "learning_rate": 0.0007534924774322969, "loss": 1.6734, "theoretical_loss": 4.325460537632483, "tokens_seen": 222953472 }, { "epoch": 0.07, "learning_rate": 0.0007534764292878636, "loss": 1.6681, "theoretical_loss": 4.325305167782352, "tokens_seen": 223019008 }, { "epoch": 0.07, "learning_rate": 0.0007534603811434303, "loss": 1.6635, "theoretical_loss": 4.325149856361845, "tokens_seen": 223084544 }, { "epoch": 0.07, "learning_rate": 0.000753444332998997, "loss": 1.6284, "theoretical_loss": 4.3249946033318345, "tokens_seen": 223150080 }, { "epoch": 0.07, "learning_rate": 0.0007534282848545637, "loss": 1.7733, "theoretical_loss": 4.32483940865323, "tokens_seen": 223215616 }, { "epoch": 0.07, "learning_rate": 0.0007534122367101304, "loss": 1.6484, "theoretical_loss": 4.324684272286978, "tokens_seen": 223281152 }, { "epoch": 0.07, "learning_rate": 0.0007533961885656971, "loss": 1.6395, "theoretical_loss": 4.324529194194062, "tokens_seen": 223346688 }, { "epoch": 0.07, "learning_rate": 0.0007533801404212638, "loss": 1.6507, "theoretical_loss": 4.324374174335503, "tokens_seen": 223412224 }, { "epoch": 0.07, "learning_rate": 0.0007533640922768305, "loss": 1.7281, "theoretical_loss": 4.324219212672363, "tokens_seen": 223477760 }, { "epoch": 0.07, "learning_rate": 0.0007533480441323972, "loss": 1.7564, "theoretical_loss": 4.324064309165734, "tokens_seen": 223543296 }, { "epoch": 0.07, "learning_rate": 0.0007533319959879639, "loss": 1.5957, "theoretical_loss": 4.3239094637767534, "tokens_seen": 223608832 }, { "epoch": 0.07, "learning_rate": 0.0007533159478435306, "loss": 1.7318, "theoretical_loss": 4.323754676466592, "tokens_seen": 223674368 }, { "epoch": 0.07, "learning_rate": 0.0007532998996990973, "loss": 1.7811, "theoretical_loss": 4.323599947196458, "tokens_seen": 223739904 }, { "epoch": 0.07, "learning_rate": 0.000753283851554664, "loss": 1.7292, "theoretical_loss": 4.323445275927597, "tokens_seen": 223805440 }, { "epoch": 0.07, "learning_rate": 0.0007532678034102307, "loss": 1.712, "theoretical_loss": 4.323290662621292, "tokens_seen": 223870976 }, { "epoch": 0.07, "learning_rate": 0.0007532517552657974, "loss": 1.7663, "theoretical_loss": 4.323136107238865, "tokens_seen": 223936512 }, { "epoch": 0.07, "learning_rate": 0.0007532357071213641, "loss": 1.7012, "theoretical_loss": 4.322981609741671, "tokens_seen": 224002048 }, { "epoch": 0.07, "learning_rate": 0.0007532196589769308, "loss": 1.6102, "theoretical_loss": 4.322827170091106, "tokens_seen": 224067584 }, { "epoch": 0.07, "learning_rate": 0.0007532036108324975, "loss": 1.6167, "theoretical_loss": 4.322672788248601, "tokens_seen": 224133120 }, { "epoch": 0.07, "learning_rate": 0.0007531875626880642, "loss": 1.5478, "theoretical_loss": 4.3225184641756265, "tokens_seen": 224198656 }, { "epoch": 0.07, "learning_rate": 0.0007531715145436309, "loss": 1.6447, "theoretical_loss": 4.322364197833684, "tokens_seen": 224264192 }, { "epoch": 0.07, "learning_rate": 0.0007531554663991976, "loss": 1.618, "theoretical_loss": 4.322209989184319, "tokens_seen": 224329728 }, { "epoch": 0.07, "learning_rate": 0.0007531394182547644, "loss": 1.6174, "theoretical_loss": 4.32205583818911, "tokens_seen": 224395264 }, { "epoch": 0.07, "learning_rate": 0.000753123370110331, "loss": 1.6718, "theoretical_loss": 4.321901744809672, "tokens_seen": 224460800 }, { "epoch": 0.07, "learning_rate": 0.0007531073219658978, "loss": 1.6626, "theoretical_loss": 4.321747709007658, "tokens_seen": 224526336 }, { "epoch": 0.07, "learning_rate": 0.0007530912738214644, "loss": 1.6943, "theoretical_loss": 4.321593730744759, "tokens_seen": 224591872 }, { "epoch": 0.07, "learning_rate": 0.0007530752256770312, "loss": 1.7384, "theoretical_loss": 4.3214398099826985, "tokens_seen": 224657408 }, { "epoch": 0.07, "learning_rate": 0.0007530591775325978, "loss": 1.6295, "theoretical_loss": 4.321285946683242, "tokens_seen": 224722944 }, { "epoch": 0.07, "learning_rate": 0.0007530431293881646, "loss": 1.6245, "theoretical_loss": 4.321132140808186, "tokens_seen": 224788480 }, { "epoch": 0.07, "learning_rate": 0.0007530270812437312, "loss": 1.6358, "theoretical_loss": 4.320978392319368, "tokens_seen": 224854016 }, { "epoch": 0.07, "learning_rate": 0.000753011033099298, "loss": 1.6467, "theoretical_loss": 4.320824701178661, "tokens_seen": 224919552 }, { "epoch": 0.07, "learning_rate": 0.0007529949849548646, "loss": 1.6154, "theoretical_loss": 4.320671067347972, "tokens_seen": 224985088 }, { "epoch": 0.07, "learning_rate": 0.0007529789368104314, "loss": 1.7091, "theoretical_loss": 4.320517490789246, "tokens_seen": 225050624 }, { "epoch": 0.07, "learning_rate": 0.000752962888665998, "loss": 1.5441, "theoretical_loss": 4.320363971464466, "tokens_seen": 225116160 }, { "epoch": 0.07, "learning_rate": 0.0007529468405215648, "loss": 1.913, "theoretical_loss": 4.320210509335649, "tokens_seen": 225181696 }, { "epoch": 0.07, "learning_rate": 0.0007529307923771314, "loss": 1.7173, "theoretical_loss": 4.320057104364848, "tokens_seen": 225247232 }, { "epoch": 0.07, "learning_rate": 0.0007529147442326982, "loss": 1.5435, "theoretical_loss": 4.3199037565141545, "tokens_seen": 225312768 }, { "epoch": 0.07, "learning_rate": 0.0007528986960882648, "loss": 1.5532, "theoretical_loss": 4.319750465745694, "tokens_seen": 225378304 }, { "epoch": 0.07, "learning_rate": 0.0007528826479438316, "loss": 1.5809, "theoretical_loss": 4.31959723202163, "tokens_seen": 225443840 }, { "epoch": 0.07, "learning_rate": 0.0007528665997993982, "loss": 1.6615, "theoretical_loss": 4.319444055304161, "tokens_seen": 225509376 }, { "epoch": 0.07, "learning_rate": 0.000752850551654965, "loss": 1.6242, "theoretical_loss": 4.319290935555521, "tokens_seen": 225574912 }, { "epoch": 0.07, "learning_rate": 0.0007528345035105316, "loss": 1.6218, "theoretical_loss": 4.319137872737979, "tokens_seen": 225640448 }, { "epoch": 0.07, "learning_rate": 0.0007528184553660984, "loss": 1.7047, "theoretical_loss": 4.318984866813844, "tokens_seen": 225705984 }, { "epoch": 0.07, "learning_rate": 0.000752802407221665, "loss": 1.678, "theoretical_loss": 4.318831917745458, "tokens_seen": 225771520 }, { "epoch": 0.07, "learning_rate": 0.0007527863590772318, "loss": 1.6995, "theoretical_loss": 4.318679025495197, "tokens_seen": 225837056 }, { "epoch": 0.07, "learning_rate": 0.0007527703109327984, "loss": 1.6975, "theoretical_loss": 4.318526190025478, "tokens_seen": 225902592 }, { "epoch": 0.07, "learning_rate": 0.0007527542627883652, "loss": 1.7384, "theoretical_loss": 4.318373411298749, "tokens_seen": 225968128 }, { "epoch": 0.07, "learning_rate": 0.0007527382146439318, "loss": 1.6878, "theoretical_loss": 4.318220689277496, "tokens_seen": 226033664 }, { "epoch": 0.07, "objective/train/docs_used": 81462, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.65253746509552, "objective/train/theoretical_loss": 4.318068023924241, "objective/train/tokens_used": 246559200, "theoretical_loss": 4.318068023924241, "tokens_seen": 226099200 }, { "epoch": 0.07, "learning_rate": 0.0007527221664994986, "loss": 1.7266, "theoretical_loss": 4.318068023924241, "tokens_seen": 226099200 }, { "epoch": 0.07, "learning_rate": 0.0007527061183550652, "loss": 1.536, "theoretical_loss": 4.317915415201538, "tokens_seen": 226164736 }, { "epoch": 0.07, "learning_rate": 0.000752690070210632, "loss": 1.512, "theoretical_loss": 4.317762863071982, "tokens_seen": 226230272 }, { "epoch": 0.07, "learning_rate": 0.0007526740220661986, "loss": 1.5732, "theoretical_loss": 4.317610367498201, "tokens_seen": 226295808 }, { "epoch": 0.07, "learning_rate": 0.0007526579739217654, "loss": 1.7787, "theoretical_loss": 4.317457928442856, "tokens_seen": 226361344 }, { "epoch": 0.07, "learning_rate": 0.000752641925777332, "loss": 1.6781, "theoretical_loss": 4.317305545868648, "tokens_seen": 226426880 }, { "epoch": 0.07, "learning_rate": 0.0007526258776328988, "loss": 1.7043, "theoretical_loss": 4.317153219738311, "tokens_seen": 226492416 }, { "epoch": 0.07, "learning_rate": 0.0007526098294884654, "loss": 1.6454, "theoretical_loss": 4.317000950014615, "tokens_seen": 226557952 }, { "epoch": 0.07, "learning_rate": 0.0007525937813440322, "loss": 1.5971, "theoretical_loss": 4.316848736660363, "tokens_seen": 226623488 }, { "epoch": 0.07, "learning_rate": 0.0007525777331995988, "loss": 1.6237, "theoretical_loss": 4.316696579638396, "tokens_seen": 226689024 }, { "epoch": 0.07, "learning_rate": 0.0007525616850551656, "loss": 1.6166, "theoretical_loss": 4.316544478911592, "tokens_seen": 226754560 }, { "epoch": 0.07, "learning_rate": 0.0007525456369107322, "loss": 1.7207, "theoretical_loss": 4.316392434442858, "tokens_seen": 226820096 }, { "epoch": 0.07, "learning_rate": 0.000752529588766299, "loss": 1.6948, "theoretical_loss": 4.316240446195142, "tokens_seen": 226885632 }, { "epoch": 0.07, "learning_rate": 0.0007525135406218656, "loss": 1.6371, "theoretical_loss": 4.3160885141314225, "tokens_seen": 226951168 }, { "epoch": 0.07, "learning_rate": 0.0007524974924774324, "loss": 1.6582, "theoretical_loss": 4.315936638214718, "tokens_seen": 227016704 }, { "epoch": 0.07, "learning_rate": 0.000752481444332999, "loss": 1.6293, "theoretical_loss": 4.315784818408079, "tokens_seen": 227082240 }, { "epoch": 0.07, "learning_rate": 0.0007524653961885658, "loss": 1.6876, "theoretical_loss": 4.31563305467459, "tokens_seen": 227147776 }, { "epoch": 0.07, "learning_rate": 0.0007524493480441324, "loss": 1.6117, "theoretical_loss": 4.315481346977374, "tokens_seen": 227213312 }, { "epoch": 0.07, "learning_rate": 0.0007524332998996992, "loss": 1.5356, "theoretical_loss": 4.315329695279584, "tokens_seen": 227278848 }, { "epoch": 0.07, "learning_rate": 0.0007524172517552658, "loss": 1.7156, "theoretical_loss": 4.315178099544413, "tokens_seen": 227344384 }, { "epoch": 0.07, "learning_rate": 0.0007524012036108326, "loss": 1.5758, "theoretical_loss": 4.315026559735085, "tokens_seen": 227409920 }, { "epoch": 0.07, "learning_rate": 0.0007523851554663992, "loss": 1.7364, "theoretical_loss": 4.314875075814861, "tokens_seen": 227475456 }, { "epoch": 0.07, "learning_rate": 0.0007523691073219659, "loss": 1.716, "theoretical_loss": 4.314723647747035, "tokens_seen": 227540992 }, { "epoch": 0.07, "learning_rate": 0.0007523530591775327, "loss": 1.5991, "theoretical_loss": 4.314572275494937, "tokens_seen": 227606528 }, { "epoch": 0.07, "learning_rate": 0.0007523370110330993, "loss": 1.4546, "theoretical_loss": 4.3144209590219305, "tokens_seen": 227672064 }, { "epoch": 0.07, "learning_rate": 0.000752320962888666, "loss": 1.6459, "theoretical_loss": 4.314269698291415, "tokens_seen": 227737600 }, { "epoch": 0.07, "learning_rate": 0.0007523049147442327, "loss": 1.7276, "theoretical_loss": 4.3141184932668235, "tokens_seen": 227803136 }, { "epoch": 0.07, "learning_rate": 0.0007522888665997995, "loss": 1.6812, "theoretical_loss": 4.3139673439116235, "tokens_seen": 227868672 }, { "epoch": 0.07, "learning_rate": 0.0007522728184553661, "loss": 1.7196, "theoretical_loss": 4.313816250189317, "tokens_seen": 227934208 }, { "epoch": 0.07, "learning_rate": 0.0007522567703109329, "loss": 1.6224, "theoretical_loss": 4.313665212063441, "tokens_seen": 227999744 }, { "epoch": 0.07, "learning_rate": 0.0007522407221664995, "loss": 1.6276, "theoretical_loss": 4.313514229497566, "tokens_seen": 228065280 }, { "epoch": 0.07, "learning_rate": 0.0007522246740220661, "loss": 1.5987, "theoretical_loss": 4.313363302455299, "tokens_seen": 228130816 }, { "epoch": 0.07, "learning_rate": 0.0007522086258776329, "loss": 1.7328, "theoretical_loss": 4.313212430900277, "tokens_seen": 228196352 }, { "epoch": 0.07, "learning_rate": 0.0007521925777331995, "loss": 1.5894, "theoretical_loss": 4.313061614796176, "tokens_seen": 228261888 }, { "epoch": 0.07, "learning_rate": 0.0007521765295887663, "loss": 1.766, "theoretical_loss": 4.312910854106702, "tokens_seen": 228327424 }, { "epoch": 0.07, "learning_rate": 0.000752160481444333, "loss": 1.6051, "theoretical_loss": 4.3127601487956, "tokens_seen": 228392960 }, { "epoch": 0.07, "learning_rate": 0.0007521444332998997, "loss": 1.7288, "theoretical_loss": 4.312609498826644, "tokens_seen": 228458496 }, { "epoch": 0.07, "learning_rate": 0.0007521283851554664, "loss": 1.5024, "theoretical_loss": 4.312458904163645, "tokens_seen": 228524032 }, { "epoch": 0.07, "learning_rate": 0.0007521123370110331, "loss": 1.7327, "theoretical_loss": 4.312308364770448, "tokens_seen": 228589568 }, { "epoch": 0.07, "learning_rate": 0.0007520962888665998, "loss": 1.5368, "theoretical_loss": 4.312157880610931, "tokens_seen": 228655104 }, { "epoch": 0.07, "learning_rate": 0.0007520802407221665, "loss": 1.5935, "theoretical_loss": 4.312007451649006, "tokens_seen": 228720640 }, { "epoch": 0.07, "learning_rate": 0.0007520641925777332, "loss": 1.6157, "theoretical_loss": 4.31185707784862, "tokens_seen": 228786176 }, { "epoch": 0.07, "learning_rate": 0.0007520481444332999, "loss": 1.5835, "theoretical_loss": 4.311706759173753, "tokens_seen": 228851712 }, { "epoch": 0.07, "learning_rate": 0.0007520320962888666, "loss": 1.5946, "theoretical_loss": 4.311556495588419, "tokens_seen": 228917248 }, { "epoch": 0.07, "learning_rate": 0.0007520160481444333, "loss": 1.6795, "theoretical_loss": 4.3114062870566645, "tokens_seen": 228982784 }, { "epoch": 0.07, "learning_rate": 0.000752, "loss": 1.5991, "theoretical_loss": 4.311256133542573, "tokens_seen": 229048320 }, { "epoch": 0.07, "learning_rate": 0.0007519839518555667, "loss": 1.6723, "theoretical_loss": 4.311106035010259, "tokens_seen": 229113856 }, { "epoch": 0.07, "learning_rate": 0.0007519679037111334, "loss": 1.5913, "theoretical_loss": 4.310955991423871, "tokens_seen": 229179392 }, { "epoch": 0.07, "learning_rate": 0.0007519518555667001, "loss": 1.6719, "theoretical_loss": 4.310806002747592, "tokens_seen": 229244928 }, { "epoch": 0.07, "learning_rate": 0.0007519358074222669, "loss": 1.5921, "theoretical_loss": 4.310656068945637, "tokens_seen": 229310464 }, { "epoch": 0.07, "objective/train/docs_used": 82681, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5853383541107178, "objective/train/theoretical_loss": 4.310506189982257, "objective/train/tokens_used": 249836000, "theoretical_loss": 4.310506189982257, "tokens_seen": 229376000 }, { "epoch": 0.07, "learning_rate": 0.0007519197592778335, "loss": 1.601, "theoretical_loss": 4.310506189982257, "tokens_seen": 229376000 }, { "epoch": 0.07, "learning_rate": 0.0007519037111334003, "loss": 1.7332, "theoretical_loss": 4.310356365821734, "tokens_seen": 229441536 }, { "epoch": 0.07, "learning_rate": 0.0007518876629889669, "loss": 1.7276, "theoretical_loss": 4.310206596428387, "tokens_seen": 229507072 }, { "epoch": 0.07, "learning_rate": 0.0007518716148445337, "loss": 1.6907, "theoretical_loss": 4.310056881766561, "tokens_seen": 229572608 }, { "epoch": 0.07, "learning_rate": 0.0007518555667001003, "loss": 1.6095, "theoretical_loss": 4.309907221800645, "tokens_seen": 229638144 }, { "epoch": 0.07, "learning_rate": 0.0007518395185556671, "loss": 1.7901, "theoretical_loss": 4.309757616495053, "tokens_seen": 229703680 }, { "epoch": 0.07, "learning_rate": 0.0007518234704112337, "loss": 1.6816, "theoretical_loss": 4.309608065814234, "tokens_seen": 229769216 }, { "epoch": 0.07, "learning_rate": 0.0007518074222668005, "loss": 1.5572, "theoretical_loss": 4.309458569722673, "tokens_seen": 229834752 }, { "epoch": 0.07, "learning_rate": 0.0007517913741223671, "loss": 1.7167, "theoretical_loss": 4.309309128184886, "tokens_seen": 229900288 }, { "epoch": 0.07, "learning_rate": 0.0007517753259779339, "loss": 1.6913, "theoretical_loss": 4.309159741165422, "tokens_seen": 229965824 }, { "epoch": 0.07, "learning_rate": 0.0007517592778335005, "loss": 1.6231, "theoretical_loss": 4.309010408628865, "tokens_seen": 230031360 }, { "epoch": 0.07, "learning_rate": 0.0007517432296890673, "loss": 1.5985, "theoretical_loss": 4.308861130539829, "tokens_seen": 230096896 }, { "epoch": 0.07, "learning_rate": 0.0007517271815446339, "loss": 1.6563, "theoretical_loss": 4.308711906862965, "tokens_seen": 230162432 }, { "epoch": 0.07, "learning_rate": 0.0007517111334002007, "loss": 1.6262, "theoretical_loss": 4.308562737562953, "tokens_seen": 230227968 }, { "epoch": 0.07, "learning_rate": 0.0007516950852557673, "loss": 1.7089, "theoretical_loss": 4.30841362260451, "tokens_seen": 230293504 }, { "epoch": 0.07, "learning_rate": 0.0007516790371113341, "loss": 1.612, "theoretical_loss": 4.308264561952381, "tokens_seen": 230359040 }, { "epoch": 0.07, "learning_rate": 0.0007516629889669007, "loss": 1.6092, "theoretical_loss": 4.30811555557135, "tokens_seen": 230424576 }, { "epoch": 0.07, "learning_rate": 0.0007516469408224675, "loss": 1.6265, "theoretical_loss": 4.307966603426227, "tokens_seen": 230490112 }, { "epoch": 0.07, "learning_rate": 0.0007516308926780341, "loss": 1.7801, "theoretical_loss": 4.307817705481861, "tokens_seen": 230555648 }, { "epoch": 0.07, "learning_rate": 0.0007516148445336009, "loss": 1.7435, "theoretical_loss": 4.307668861703131, "tokens_seen": 230621184 }, { "epoch": 0.07, "learning_rate": 0.0007515987963891675, "loss": 1.6081, "theoretical_loss": 4.307520072054947, "tokens_seen": 230686720 }, { "epoch": 0.07, "learning_rate": 0.0007515827482447343, "loss": 1.5812, "theoretical_loss": 4.307371336502254, "tokens_seen": 230752256 }, { "epoch": 0.07, "learning_rate": 0.0007515667001003009, "loss": 1.6561, "theoretical_loss": 4.307222655010031, "tokens_seen": 230817792 }, { "epoch": 0.07, "learning_rate": 0.0007515506519558677, "loss": 1.6483, "theoretical_loss": 4.307074027543287, "tokens_seen": 230883328 }, { "epoch": 0.07, "learning_rate": 0.0007515346038114343, "loss": 1.7229, "theoretical_loss": 4.3069254540670645, "tokens_seen": 230948864 }, { "epoch": 0.07, "learning_rate": 0.0007515185556670011, "loss": 1.7828, "theoretical_loss": 4.306776934546436, "tokens_seen": 231014400 }, { "epoch": 0.07, "learning_rate": 0.0007515025075225677, "loss": 1.5098, "theoretical_loss": 4.306628468946512, "tokens_seen": 231079936 }, { "epoch": 0.07, "learning_rate": 0.0007514864593781345, "loss": 1.55, "theoretical_loss": 4.306480057232431, "tokens_seen": 231145472 }, { "epoch": 0.07, "learning_rate": 0.0007514704112337011, "loss": 1.6725, "theoretical_loss": 4.306331699369366, "tokens_seen": 231211008 }, { "epoch": 0.07, "learning_rate": 0.0007514543630892679, "loss": 1.6515, "theoretical_loss": 4.30618339532252, "tokens_seen": 231276544 }, { "epoch": 0.07, "learning_rate": 0.0007514383149448345, "loss": 1.7387, "theoretical_loss": 4.306035145057132, "tokens_seen": 231342080 }, { "epoch": 0.07, "learning_rate": 0.0007514222668004013, "loss": 1.6858, "theoretical_loss": 4.30588694853847, "tokens_seen": 231407616 }, { "epoch": 0.07, "learning_rate": 0.0007514062186559679, "loss": 1.6754, "theoretical_loss": 4.305738805731836, "tokens_seen": 231473152 }, { "epoch": 0.07, "learning_rate": 0.0007513901705115347, "loss": 1.6655, "theoretical_loss": 4.305590716602563, "tokens_seen": 231538688 }, { "epoch": 0.07, "learning_rate": 0.0007513741223671013, "loss": 1.6448, "theoretical_loss": 4.305442681116018, "tokens_seen": 231604224 }, { "epoch": 0.07, "learning_rate": 0.0007513580742226681, "loss": 1.6165, "theoretical_loss": 4.305294699237598, "tokens_seen": 231669760 }, { "epoch": 0.07, "learning_rate": 0.0007513420260782347, "loss": 1.5662, "theoretical_loss": 4.305146770932734, "tokens_seen": 231735296 }, { "epoch": 0.07, "learning_rate": 0.0007513259779338015, "loss": 1.6911, "theoretical_loss": 4.304998896166888, "tokens_seen": 231800832 }, { "epoch": 0.07, "learning_rate": 0.0007513099297893681, "loss": 1.709, "theoretical_loss": 4.304851074905554, "tokens_seen": 231866368 }, { "epoch": 0.07, "learning_rate": 0.0007512938816449349, "loss": 1.6598, "theoretical_loss": 4.304703307114258, "tokens_seen": 231931904 }, { "epoch": 0.07, "learning_rate": 0.0007512778335005015, "loss": 1.6776, "theoretical_loss": 4.304555592758559, "tokens_seen": 231997440 }, { "epoch": 0.07, "learning_rate": 0.0007512617853560683, "loss": 1.5082, "theoretical_loss": 4.304407931804047, "tokens_seen": 232062976 }, { "epoch": 0.07, "learning_rate": 0.0007512457372116349, "loss": 1.6469, "theoretical_loss": 4.304260324216344, "tokens_seen": 232128512 }, { "epoch": 0.07, "learning_rate": 0.0007512296890672017, "loss": 1.6334, "theoretical_loss": 4.304112769961102, "tokens_seen": 232194048 }, { "epoch": 0.07, "learning_rate": 0.0007512136409227683, "loss": 1.532, "theoretical_loss": 4.303965269004008, "tokens_seen": 232259584 }, { "epoch": 0.07, "learning_rate": 0.0007511975927783351, "loss": 1.7242, "theoretical_loss": 4.30381782131078, "tokens_seen": 232325120 }, { "epoch": 0.07, "learning_rate": 0.0007511815446339017, "loss": 1.6597, "theoretical_loss": 4.303670426847166, "tokens_seen": 232390656 }, { "epoch": 0.07, "learning_rate": 0.0007511654964894685, "loss": 1.5863, "theoretical_loss": 4.303523085578946, "tokens_seen": 232456192 }, { "epoch": 0.07, "learning_rate": 0.0007511494483450351, "loss": 1.6315, "theoretical_loss": 4.303375797471935, "tokens_seen": 232521728 }, { "epoch": 0.07, "learning_rate": 0.0007511334002006019, "loss": 1.6744, "theoretical_loss": 4.303228562491974, "tokens_seen": 232587264 }, { "epoch": 0.07, "objective/train/docs_used": 83903, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6346615552902222, "objective/train/theoretical_loss": 4.303081380604939, "objective/train/tokens_used": 253112800, "theoretical_loss": 4.303081380604939, "tokens_seen": 232652800 }, { "epoch": 0.07, "learning_rate": 0.0007511173520561685, "loss": 1.6698, "theoretical_loss": 4.303081380604939, "tokens_seen": 232652800 }, { "epoch": 0.07, "learning_rate": 0.0007511013039117353, "loss": 1.5696, "theoretical_loss": 4.302934251776739, "tokens_seen": 232718336 }, { "epoch": 0.07, "learning_rate": 0.000751085255767302, "loss": 1.7224, "theoretical_loss": 4.302787175973311, "tokens_seen": 232783872 }, { "epoch": 0.07, "learning_rate": 0.0007510692076228687, "loss": 1.7526, "theoretical_loss": 4.302640153160625, "tokens_seen": 232849408 }, { "epoch": 0.07, "learning_rate": 0.0007510531594784353, "loss": 1.6315, "theoretical_loss": 4.302493183304683, "tokens_seen": 232914944 }, { "epoch": 0.07, "learning_rate": 0.0007510371113340021, "loss": 1.6435, "theoretical_loss": 4.302346266371517, "tokens_seen": 232980480 }, { "epoch": 0.07, "learning_rate": 0.0007510210631895687, "loss": 1.6499, "theoretical_loss": 4.302199402327191, "tokens_seen": 233046016 }, { "epoch": 0.07, "learning_rate": 0.0007510050150451355, "loss": 1.6708, "theoretical_loss": 4.302052591137802, "tokens_seen": 233111552 }, { "epoch": 0.07, "learning_rate": 0.0007509889669007021, "loss": 1.6101, "theoretical_loss": 4.301905832769474, "tokens_seen": 233177088 }, { "epoch": 0.07, "learning_rate": 0.0007509729187562688, "loss": 1.6334, "theoretical_loss": 4.301759127188367, "tokens_seen": 233242624 }, { "epoch": 0.07, "learning_rate": 0.0007509568706118356, "loss": 1.7197, "theoretical_loss": 4.301612474360669, "tokens_seen": 233308160 }, { "epoch": 0.07, "learning_rate": 0.0007509408224674022, "loss": 1.7176, "theoretical_loss": 4.301465874252601, "tokens_seen": 233373696 }, { "epoch": 0.07, "learning_rate": 0.000750924774322969, "loss": 1.7187, "theoretical_loss": 4.301319326830413, "tokens_seen": 233439232 }, { "epoch": 0.07, "learning_rate": 0.0007509087261785356, "loss": 1.7268, "theoretical_loss": 4.301172832060389, "tokens_seen": 233504768 }, { "epoch": 0.07, "learning_rate": 0.0007508926780341024, "loss": 1.6295, "theoretical_loss": 4.3010263899088415, "tokens_seen": 233570304 }, { "epoch": 0.07, "learning_rate": 0.000750876629889669, "loss": 1.6408, "theoretical_loss": 4.300880000342114, "tokens_seen": 233635840 }, { "epoch": 0.07, "learning_rate": 0.0007508605817452358, "loss": 1.6239, "theoretical_loss": 4.300733663326583, "tokens_seen": 233701376 }, { "epoch": 0.07, "learning_rate": 0.0007508445336008024, "loss": 1.5342, "theoretical_loss": 4.300587378828655, "tokens_seen": 233766912 }, { "epoch": 0.07, "learning_rate": 0.000750828485456369, "loss": 1.7235, "theoretical_loss": 4.300441146814766, "tokens_seen": 233832448 }, { "epoch": 0.07, "learning_rate": 0.0007508124373119358, "loss": 1.619, "theoretical_loss": 4.300294967251386, "tokens_seen": 233897984 }, { "epoch": 0.07, "learning_rate": 0.0007507963891675024, "loss": 1.6433, "theoretical_loss": 4.300148840105011, "tokens_seen": 233963520 }, { "epoch": 0.07, "learning_rate": 0.0007507803410230692, "loss": 1.5639, "theoretical_loss": 4.300002765342173, "tokens_seen": 234029056 }, { "epoch": 0.07, "learning_rate": 0.0007507642928786359, "loss": 1.672, "theoretical_loss": 4.299856742929431, "tokens_seen": 234094592 }, { "epoch": 0.07, "learning_rate": 0.0007507482447342026, "loss": 1.6359, "theoretical_loss": 4.299710772833377, "tokens_seen": 234160128 }, { "epoch": 0.07, "learning_rate": 0.0007507321965897694, "loss": 1.7174, "theoretical_loss": 4.299564855020631, "tokens_seen": 234225664 }, { "epoch": 0.07, "learning_rate": 0.000750716148445336, "loss": 1.5812, "theoretical_loss": 4.299418989457848, "tokens_seen": 234291200 }, { "epoch": 0.07, "learning_rate": 0.0007507001003009028, "loss": 1.6371, "theoretical_loss": 4.299273176111709, "tokens_seen": 234356736 }, { "epoch": 0.07, "learning_rate": 0.0007506840521564694, "loss": 1.6428, "theoretical_loss": 4.299127414948927, "tokens_seen": 234422272 }, { "epoch": 0.07, "learning_rate": 0.0007506680040120362, "loss": 1.654, "theoretical_loss": 4.298981705936248, "tokens_seen": 234487808 }, { "epoch": 0.07, "learning_rate": 0.0007506519558676028, "loss": 1.7671, "theoretical_loss": 4.298836049040444, "tokens_seen": 234553344 }, { "epoch": 0.07, "learning_rate": 0.0007506359077231696, "loss": 1.7702, "theoretical_loss": 4.298690444228322, "tokens_seen": 234618880 }, { "epoch": 0.07, "learning_rate": 0.0007506198595787362, "loss": 1.5735, "theoretical_loss": 4.298544891466717, "tokens_seen": 234684416 }, { "epoch": 0.07, "learning_rate": 0.000750603811434303, "loss": 1.6077, "theoretical_loss": 4.298399390722493, "tokens_seen": 234749952 }, { "epoch": 0.07, "learning_rate": 0.0007505877632898696, "loss": 1.725, "theoretical_loss": 4.298253941962548, "tokens_seen": 234815488 }, { "epoch": 0.07, "learning_rate": 0.0007505717151454364, "loss": 1.5374, "theoretical_loss": 4.2981085451538075, "tokens_seen": 234881024 }, { "epoch": 0.07, "learning_rate": 0.000750555667001003, "loss": 1.6508, "theoretical_loss": 4.297963200263228, "tokens_seen": 234946560 }, { "epoch": 0.07, "learning_rate": 0.0007505396188565698, "loss": 1.7077, "theoretical_loss": 4.297817907257797, "tokens_seen": 235012096 }, { "epoch": 0.07, "learning_rate": 0.0007505235707121364, "loss": 1.6419, "theoretical_loss": 4.29767266610453, "tokens_seen": 235077632 }, { "epoch": 0.07, "learning_rate": 0.0007505075225677032, "loss": 1.6664, "theoretical_loss": 4.297527476770476, "tokens_seen": 235143168 }, { "epoch": 0.07, "learning_rate": 0.0007504914744232698, "loss": 1.6296, "theoretical_loss": 4.297382339222711, "tokens_seen": 235208704 }, { "epoch": 0.07, "learning_rate": 0.0007504754262788366, "loss": 1.693, "theoretical_loss": 4.297237253428342, "tokens_seen": 235274240 }, { "epoch": 0.07, "learning_rate": 0.0007504593781344032, "loss": 1.8014, "theoretical_loss": 4.297092219354509, "tokens_seen": 235339776 }, { "epoch": 0.07, "learning_rate": 0.00075044332998997, "loss": 1.7015, "theoretical_loss": 4.296947236968376, "tokens_seen": 235405312 }, { "epoch": 0.07, "learning_rate": 0.0007504272818455366, "loss": 1.6181, "theoretical_loss": 4.296802306237143, "tokens_seen": 235470848 }, { "epoch": 0.07, "learning_rate": 0.0007504112337011034, "loss": 1.7072, "theoretical_loss": 4.296657427128036, "tokens_seen": 235536384 }, { "epoch": 0.07, "learning_rate": 0.00075039518555667, "loss": 1.629, "theoretical_loss": 4.296512599608311, "tokens_seen": 235601920 }, { "epoch": 0.07, "learning_rate": 0.0007503791374122368, "loss": 1.6502, "theoretical_loss": 4.296367823645259, "tokens_seen": 235667456 }, { "epoch": 0.07, "learning_rate": 0.0007503630892678034, "loss": 1.671, "theoretical_loss": 4.296223099206193, "tokens_seen": 235732992 }, { "epoch": 0.07, "learning_rate": 0.0007503470411233702, "loss": 1.7477, "theoretical_loss": 4.296078426258462, "tokens_seen": 235798528 }, { "epoch": 0.07, "learning_rate": 0.0007503309929789368, "loss": 1.521, "theoretical_loss": 4.295933804769442, "tokens_seen": 235864064 }, { "epoch": 0.07, "objective/train/docs_used": 84743, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6421600580215454, "objective/train/theoretical_loss": 4.295789234706538, "objective/train/tokens_used": 256389600, "theoretical_loss": 4.295789234706538, "tokens_seen": 235929600 }, { "epoch": 0.07, "learning_rate": 0.0007503149448345036, "loss": 1.7047, "theoretical_loss": 4.295789234706538, "tokens_seen": 235929600 }, { "epoch": 0.07, "learning_rate": 0.0007502988966900702, "loss": 1.7024, "theoretical_loss": 4.295644716037188, "tokens_seen": 235995136 }, { "epoch": 0.07, "learning_rate": 0.000750282848545637, "loss": 1.7471, "theoretical_loss": 4.295500248728856, "tokens_seen": 236060672 }, { "epoch": 0.07, "learning_rate": 0.0007502668004012036, "loss": 1.7045, "theoretical_loss": 4.295355832749038, "tokens_seen": 236126208 }, { "epoch": 0.07, "learning_rate": 0.0007502507522567704, "loss": 1.7213, "theoretical_loss": 4.295211468065258, "tokens_seen": 236191744 }, { "epoch": 0.07, "learning_rate": 0.000750234704112337, "loss": 1.715, "theoretical_loss": 4.295067154645071, "tokens_seen": 236257280 }, { "epoch": 0.07, "learning_rate": 0.0007502186559679038, "loss": 1.6261, "theoretical_loss": 4.294922892456061, "tokens_seen": 236322816 }, { "epoch": 0.07, "learning_rate": 0.0007502026078234704, "loss": 1.601, "theoretical_loss": 4.294778681465841, "tokens_seen": 236388352 }, { "epoch": 0.07, "learning_rate": 0.0007501865596790372, "loss": 1.4909, "theoretical_loss": 4.294634521642053, "tokens_seen": 236453888 }, { "epoch": 0.07, "learning_rate": 0.0007501705115346038, "loss": 1.7144, "theoretical_loss": 4.294490412952371, "tokens_seen": 236519424 }, { "epoch": 0.07, "learning_rate": 0.0007501544633901706, "loss": 1.7742, "theoretical_loss": 4.294346355364494, "tokens_seen": 236584960 }, { "epoch": 0.07, "learning_rate": 0.0007501384152457372, "loss": 1.678, "theoretical_loss": 4.294202348846156, "tokens_seen": 236650496 }, { "epoch": 0.07, "learning_rate": 0.000750122367101304, "loss": 1.6133, "theoretical_loss": 4.294058393365114, "tokens_seen": 236716032 }, { "epoch": 0.07, "learning_rate": 0.0007501063189568706, "loss": 1.685, "theoretical_loss": 4.293914488889161, "tokens_seen": 236781568 }, { "epoch": 0.07, "learning_rate": 0.0007500902708124374, "loss": 1.6133, "theoretical_loss": 4.293770635386112, "tokens_seen": 236847104 }, { "epoch": 0.07, "learning_rate": 0.000750074222668004, "loss": 1.716, "theoretical_loss": 4.293626832823818, "tokens_seen": 236912640 }, { "epoch": 0.07, "learning_rate": 0.0007500581745235708, "loss": 1.6319, "theoretical_loss": 4.293483081170153, "tokens_seen": 236978176 }, { "epoch": 0.07, "learning_rate": 0.0007500421263791374, "loss": 1.5912, "theoretical_loss": 4.293339380393027, "tokens_seen": 237043712 }, { "epoch": 0.07, "learning_rate": 0.0007500260782347042, "loss": 1.6264, "theoretical_loss": 4.293195730460372, "tokens_seen": 237109248 }, { "epoch": 0.07, "learning_rate": 0.0007500100300902708, "loss": 1.5781, "theoretical_loss": 4.293052131340154, "tokens_seen": 237174784 }, { "epoch": 0.07, "learning_rate": 0.0007499939819458376, "loss": 1.7007, "theoretical_loss": 4.292908583000365, "tokens_seen": 237240320 }, { "epoch": 0.07, "learning_rate": 0.0007499779338014042, "loss": 1.663, "theoretical_loss": 4.29276508540903, "tokens_seen": 237305856 }, { "epoch": 0.07, "learning_rate": 0.000749961885656971, "loss": 1.5671, "theoretical_loss": 4.292621638534198, "tokens_seen": 237371392 }, { "epoch": 0.07, "learning_rate": 0.0007499458375125376, "loss": 1.7502, "theoretical_loss": 4.29247824234395, "tokens_seen": 237436928 }, { "epoch": 0.07, "learning_rate": 0.0007499297893681044, "loss": 1.715, "theoretical_loss": 4.292334896806397, "tokens_seen": 237502464 }, { "epoch": 0.07, "learning_rate": 0.000749913741223671, "loss": 1.6047, "theoretical_loss": 4.292191601889673, "tokens_seen": 237568000 }, { "epoch": 0.07, "learning_rate": 0.0007498976930792378, "loss": 1.6171, "theoretical_loss": 4.2920483575619475, "tokens_seen": 237633536 }, { "epoch": 0.07, "learning_rate": 0.0007498816449348044, "loss": 1.7442, "theoretical_loss": 4.291905163791417, "tokens_seen": 237699072 }, { "epoch": 0.07, "learning_rate": 0.0007498655967903712, "loss": 1.6901, "theoretical_loss": 4.291762020546304, "tokens_seen": 237764608 }, { "epoch": 0.07, "learning_rate": 0.0007498495486459378, "loss": 1.6801, "theoretical_loss": 4.291618927794862, "tokens_seen": 237830144 }, { "epoch": 0.07, "learning_rate": 0.0007498335005015046, "loss": 1.669, "theoretical_loss": 4.291475885505374, "tokens_seen": 237895680 }, { "epoch": 0.07, "learning_rate": 0.0007498174523570712, "loss": 1.6468, "theoretical_loss": 4.291332893646149, "tokens_seen": 237961216 }, { "epoch": 0.07, "learning_rate": 0.000749801404212638, "loss": 1.5978, "theoretical_loss": 4.291189952185527, "tokens_seen": 238026752 }, { "epoch": 0.07, "learning_rate": 0.0007497853560682046, "loss": 1.5644, "theoretical_loss": 4.291047061091875, "tokens_seen": 238092288 }, { "epoch": 0.07, "learning_rate": 0.0007497693079237714, "loss": 1.5605, "theoretical_loss": 4.290904220333591, "tokens_seen": 238157824 }, { "epoch": 0.07, "learning_rate": 0.000749753259779338, "loss": 1.6864, "theoretical_loss": 4.2907614298790975, "tokens_seen": 238223360 }, { "epoch": 0.07, "learning_rate": 0.0007497372116349048, "loss": 1.685, "theoretical_loss": 4.290618689696848, "tokens_seen": 238288896 }, { "epoch": 0.07, "learning_rate": 0.0007497211634904714, "loss": 1.6605, "theoretical_loss": 4.290475999755326, "tokens_seen": 238354432 }, { "epoch": 0.07, "learning_rate": 0.0007497051153460382, "loss": 1.7451, "theoretical_loss": 4.29033336002304, "tokens_seen": 238419968 }, { "epoch": 0.07, "learning_rate": 0.0007496890672016048, "loss": 1.5776, "theoretical_loss": 4.290190770468529, "tokens_seen": 238485504 }, { "epoch": 0.07, "learning_rate": 0.0007496730190571716, "loss": 1.6556, "theoretical_loss": 4.290048231060361, "tokens_seen": 238551040 }, { "epoch": 0.07, "learning_rate": 0.0007496569709127382, "loss": 1.5922, "theoretical_loss": 4.289905741767129, "tokens_seen": 238616576 }, { "epoch": 0.07, "learning_rate": 0.000749640922768305, "loss": 1.6411, "theoretical_loss": 4.289763302557457, "tokens_seen": 238682112 }, { "epoch": 0.07, "learning_rate": 0.0007496248746238716, "loss": 1.7339, "theoretical_loss": 4.289620913399998, "tokens_seen": 238747648 }, { "epoch": 0.07, "learning_rate": 0.0007496088264794384, "loss": 1.61, "theoretical_loss": 4.2894785742634305, "tokens_seen": 238813184 }, { "epoch": 0.07, "learning_rate": 0.000749592778335005, "loss": 1.6489, "theoretical_loss": 4.289336285116463, "tokens_seen": 238878720 }, { "epoch": 0.07, "learning_rate": 0.0007495767301905718, "loss": 1.6366, "theoretical_loss": 4.289194045927832, "tokens_seen": 238944256 }, { "epoch": 0.07, "learning_rate": 0.0007495606820461385, "loss": 1.57, "theoretical_loss": 4.289051856666303, "tokens_seen": 239009792 }, { "epoch": 0.07, "learning_rate": 0.0007495446339017051, "loss": 1.659, "theoretical_loss": 4.288909717300667, "tokens_seen": 239075328 }, { "epoch": 0.07, "learning_rate": 0.0007495285857572719, "loss": 1.6939, "theoretical_loss": 4.288767627799745, "tokens_seen": 239140864 }, { "epoch": 0.07, "objective/train/docs_used": 85669, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6066745519638062, "objective/train/theoretical_loss": 4.288625588132385, "objective/train/tokens_used": 259666400, "theoretical_loss": 4.288625588132385, "tokens_seen": 239206400 }, { "epoch": 0.07, "learning_rate": 0.0007495125376128385, "loss": 1.6276, "theoretical_loss": 4.288625588132385, "tokens_seen": 239206400 }, { "epoch": 0.07, "learning_rate": 0.0007494964894684053, "loss": 1.715, "theoretical_loss": 4.288483598267465, "tokens_seen": 239271936 }, { "epoch": 0.07, "learning_rate": 0.0007494804413239719, "loss": 1.5889, "theoretical_loss": 4.288341658173888, "tokens_seen": 239337472 }, { "epoch": 0.07, "learning_rate": 0.0007494643931795387, "loss": 1.7141, "theoretical_loss": 4.288199767820587, "tokens_seen": 239403008 }, { "epoch": 0.07, "learning_rate": 0.0007494483450351053, "loss": 1.682, "theoretical_loss": 4.2880579271765225, "tokens_seen": 239468544 }, { "epoch": 0.07, "learning_rate": 0.0007494322968906721, "loss": 1.5611, "theoretical_loss": 4.287916136210682, "tokens_seen": 239534080 }, { "epoch": 0.07, "learning_rate": 0.0007494162487462387, "loss": 1.6503, "theoretical_loss": 4.287774394892082, "tokens_seen": 239599616 }, { "epoch": 0.07, "learning_rate": 0.0007494002006018055, "loss": 1.6604, "theoretical_loss": 4.287632703189766, "tokens_seen": 239665152 }, { "epoch": 0.07, "learning_rate": 0.0007493841524573721, "loss": 1.5737, "theoretical_loss": 4.2874910610728065, "tokens_seen": 239730688 }, { "epoch": 0.07, "learning_rate": 0.0007493681043129389, "loss": 1.6437, "theoretical_loss": 4.287349468510302, "tokens_seen": 239796224 }, { "epoch": 0.07, "learning_rate": 0.0007493520561685055, "loss": 1.4903, "theoretical_loss": 4.28720792547138, "tokens_seen": 239861760 }, { "epoch": 0.07, "learning_rate": 0.0007493360080240723, "loss": 1.6124, "theoretical_loss": 4.287066431925194, "tokens_seen": 239927296 }, { "epoch": 0.07, "learning_rate": 0.0007493199598796389, "loss": 1.5895, "theoretical_loss": 4.286924987840926, "tokens_seen": 239992832 }, { "epoch": 0.07, "learning_rate": 0.0007493039117352057, "loss": 1.7164, "theoretical_loss": 4.2867835931877885, "tokens_seen": 240058368 }, { "epoch": 0.07, "learning_rate": 0.0007492878635907723, "loss": 1.6419, "theoretical_loss": 4.286642247935017, "tokens_seen": 240123904 }, { "epoch": 0.07, "learning_rate": 0.0007492718154463391, "loss": 1.6765, "theoretical_loss": 4.286500952051876, "tokens_seen": 240189440 }, { "epoch": 0.07, "learning_rate": 0.0007492557673019057, "loss": 1.5405, "theoretical_loss": 4.286359705507659, "tokens_seen": 240254976 }, { "epoch": 0.07, "learning_rate": 0.0007492397191574725, "loss": 1.558, "theoretical_loss": 4.286218508271686, "tokens_seen": 240320512 }, { "epoch": 0.07, "learning_rate": 0.0007492236710130391, "loss": 1.5783, "theoretical_loss": 4.286077360313303, "tokens_seen": 240386048 }, { "epoch": 0.07, "learning_rate": 0.0007492076228686059, "loss": 1.5213, "theoretical_loss": 4.285936261601887, "tokens_seen": 240451584 }, { "epoch": 0.07, "learning_rate": 0.0007491915747241725, "loss": 1.5629, "theoretical_loss": 4.285795212106839, "tokens_seen": 240517120 }, { "epoch": 0.07, "learning_rate": 0.0007491755265797393, "loss": 1.7373, "theoretical_loss": 4.2856542117975875, "tokens_seen": 240582656 }, { "epoch": 0.07, "learning_rate": 0.0007491594784353059, "loss": 1.7176, "theoretical_loss": 4.285513260643591, "tokens_seen": 240648192 }, { "epoch": 0.07, "learning_rate": 0.0007491434302908727, "loss": 1.6888, "theoretical_loss": 4.285372358614332, "tokens_seen": 240713728 }, { "epoch": 0.07, "learning_rate": 0.0007491273821464393, "loss": 1.6917, "theoretical_loss": 4.285231505679324, "tokens_seen": 240779264 }, { "epoch": 0.07, "learning_rate": 0.0007491113340020061, "loss": 1.6539, "theoretical_loss": 4.285090701808103, "tokens_seen": 240844800 }, { "epoch": 0.07, "learning_rate": 0.0007490952858575727, "loss": 1.612, "theoretical_loss": 4.284949946970236, "tokens_seen": 240910336 }, { "epoch": 0.07, "learning_rate": 0.0007490792377131395, "loss": 1.7414, "theoretical_loss": 4.284809241135315, "tokens_seen": 240975872 }, { "epoch": 0.07, "learning_rate": 0.0007490631895687061, "loss": 1.6637, "theoretical_loss": 4.284668584272962, "tokens_seen": 241041408 }, { "epoch": 0.07, "learning_rate": 0.0007490471414242729, "loss": 1.6183, "theoretical_loss": 4.284527976352823, "tokens_seen": 241106944 }, { "epoch": 0.07, "learning_rate": 0.0007490310932798395, "loss": 1.6879, "theoretical_loss": 4.284387417344572, "tokens_seen": 241172480 }, { "epoch": 0.07, "learning_rate": 0.0007490150451354063, "loss": 1.6791, "theoretical_loss": 4.28424690721791, "tokens_seen": 241238016 }, { "epoch": 0.07, "learning_rate": 0.0007489989969909729, "loss": 1.7164, "theoretical_loss": 4.284106445942565, "tokens_seen": 241303552 }, { "epoch": 0.07, "learning_rate": 0.0007489829488465397, "loss": 1.6794, "theoretical_loss": 4.2839660334882925, "tokens_seen": 241369088 }, { "epoch": 0.07, "learning_rate": 0.0007489669007021063, "loss": 1.4591, "theoretical_loss": 4.283825669824875, "tokens_seen": 241434624 }, { "epoch": 0.07, "learning_rate": 0.0007489508525576731, "loss": 1.6535, "theoretical_loss": 4.28368535492212, "tokens_seen": 241500160 }, { "epoch": 0.07, "learning_rate": 0.0007489348044132397, "loss": 1.6211, "theoretical_loss": 4.283545088749865, "tokens_seen": 241565696 }, { "epoch": 0.07, "learning_rate": 0.0007489187562688065, "loss": 1.6274, "theoretical_loss": 4.2834048712779715, "tokens_seen": 241631232 }, { "epoch": 0.07, "learning_rate": 0.0007489027081243731, "loss": 1.5905, "theoretical_loss": 4.28326470247633, "tokens_seen": 241696768 }, { "epoch": 0.07, "learning_rate": 0.0007488866599799399, "loss": 1.6188, "theoretical_loss": 4.283124582314856, "tokens_seen": 241762304 }, { "epoch": 0.07, "learning_rate": 0.0007488706118355065, "loss": 1.5419, "theoretical_loss": 4.282984510763493, "tokens_seen": 241827840 }, { "epoch": 0.07, "learning_rate": 0.0007488545636910733, "loss": 1.6195, "theoretical_loss": 4.282844487792209, "tokens_seen": 241893376 }, { "epoch": 0.07, "learning_rate": 0.0007488385155466399, "loss": 1.6174, "theoretical_loss": 4.282704513371002, "tokens_seen": 241958912 }, { "epoch": 0.07, "learning_rate": 0.0007488224674022067, "loss": 1.5593, "theoretical_loss": 4.282564587469896, "tokens_seen": 242024448 }, { "epoch": 0.07, "learning_rate": 0.0007488064192577733, "loss": 1.5889, "theoretical_loss": 4.282424710058938, "tokens_seen": 242089984 }, { "epoch": 0.07, "learning_rate": 0.0007487903711133401, "loss": 1.5381, "theoretical_loss": 4.282284881108206, "tokens_seen": 242155520 }, { "epoch": 0.07, "learning_rate": 0.0007487743229689067, "loss": 1.6208, "theoretical_loss": 4.282145100587802, "tokens_seen": 242221056 }, { "epoch": 0.07, "learning_rate": 0.0007487582748244735, "loss": 1.6344, "theoretical_loss": 4.282005368467856, "tokens_seen": 242286592 }, { "epoch": 0.07, "learning_rate": 0.0007487422266800401, "loss": 1.6803, "theoretical_loss": 4.2818656847185235, "tokens_seen": 242352128 }, { "epoch": 0.07, "learning_rate": 0.0007487261785356069, "loss": 1.523, "theoretical_loss": 4.281726049309987, "tokens_seen": 242417664 }, { "epoch": 0.07, "objective/train/docs_used": 86947, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.533555507659912, "objective/train/theoretical_loss": 4.281586462212455, "objective/train/tokens_used": 262943200, "theoretical_loss": 4.281586462212455, "tokens_seen": 242483200 }, { "epoch": 0.07, "learning_rate": 0.0007487101303911735, "loss": 1.5863, "theoretical_loss": 4.281586462212455, "tokens_seen": 242483200 }, { "epoch": 0.07, "learning_rate": 0.0007486940822467403, "loss": 1.6372, "theoretical_loss": 4.281446923396162, "tokens_seen": 242548736 }, { "epoch": 0.07, "learning_rate": 0.0007486780341023069, "loss": 1.6006, "theoretical_loss": 4.281307432831371, "tokens_seen": 242614272 }, { "epoch": 0.07, "learning_rate": 0.0007486619859578737, "loss": 1.5637, "theoretical_loss": 4.281167990488369, "tokens_seen": 242679808 }, { "epoch": 0.07, "learning_rate": 0.0007486459378134403, "loss": 1.6963, "theoretical_loss": 4.281028596337469, "tokens_seen": 242745344 }, { "epoch": 0.07, "learning_rate": 0.0007486298896690071, "loss": 1.5761, "theoretical_loss": 4.280889250349014, "tokens_seen": 242810880 }, { "epoch": 0.07, "learning_rate": 0.0007486138415245737, "loss": 1.5397, "theoretical_loss": 4.280749952493368, "tokens_seen": 242876416 }, { "epoch": 0.07, "learning_rate": 0.0007485977933801405, "loss": 1.6975, "theoretical_loss": 4.280610702740926, "tokens_seen": 242941952 }, { "epoch": 0.07, "learning_rate": 0.0007485817452357071, "loss": 1.5826, "theoretical_loss": 4.280471501062106, "tokens_seen": 243007488 }, { "epoch": 0.07, "learning_rate": 0.0007485656970912739, "loss": 1.7064, "theoretical_loss": 4.280332347427353, "tokens_seen": 243073024 }, { "epoch": 0.07, "learning_rate": 0.0007485496489468405, "loss": 1.6329, "theoretical_loss": 4.280193241807139, "tokens_seen": 243138560 }, { "epoch": 0.07, "learning_rate": 0.0007485336008024073, "loss": 1.5267, "theoretical_loss": 4.280054184171963, "tokens_seen": 243204096 }, { "epoch": 0.07, "learning_rate": 0.0007485175526579739, "loss": 1.6792, "theoretical_loss": 4.279915174492345, "tokens_seen": 243269632 }, { "epoch": 0.07, "learning_rate": 0.0007485015045135407, "loss": 1.638, "theoretical_loss": 4.279776212738838, "tokens_seen": 243335168 }, { "epoch": 0.07, "learning_rate": 0.0007484854563691073, "loss": 1.6731, "theoretical_loss": 4.279637298882016, "tokens_seen": 243400704 }, { "epoch": 0.07, "learning_rate": 0.0007484694082246741, "loss": 1.5625, "theoretical_loss": 4.27949843289248, "tokens_seen": 243466240 }, { "epoch": 0.07, "learning_rate": 0.0007484533600802407, "loss": 1.6964, "theoretical_loss": 4.2793596147408595, "tokens_seen": 243531776 }, { "epoch": 0.07, "learning_rate": 0.0007484373119358075, "loss": 1.6091, "theoretical_loss": 4.279220844397806, "tokens_seen": 243597312 }, { "epoch": 0.07, "learning_rate": 0.0007484212637913741, "loss": 1.6271, "theoretical_loss": 4.279082121834001, "tokens_seen": 243662848 }, { "epoch": 0.07, "learning_rate": 0.0007484052156469409, "loss": 1.6291, "theoretical_loss": 4.278943447020148, "tokens_seen": 243728384 }, { "epoch": 0.07, "learning_rate": 0.0007483891675025075, "loss": 1.7249, "theoretical_loss": 4.278804819926978, "tokens_seen": 243793920 }, { "epoch": 0.07, "learning_rate": 0.0007483731193580743, "loss": 1.656, "theoretical_loss": 4.278666240525249, "tokens_seen": 243859456 }, { "epoch": 0.07, "learning_rate": 0.0007483570712136409, "loss": 1.7174, "theoretical_loss": 4.278527708785743, "tokens_seen": 243924992 }, { "epoch": 0.07, "learning_rate": 0.0007483410230692077, "loss": 1.6751, "theoretical_loss": 4.278389224679268, "tokens_seen": 243990528 }, { "epoch": 0.07, "learning_rate": 0.0007483249749247745, "loss": 1.4917, "theoretical_loss": 4.27825078817666, "tokens_seen": 244056064 }, { "epoch": 0.07, "learning_rate": 0.0007483089267803411, "loss": 1.5895, "theoretical_loss": 4.2781123992487755, "tokens_seen": 244121600 }, { "epoch": 0.07, "learning_rate": 0.0007482928786359077, "loss": 1.6703, "theoretical_loss": 4.277974057866503, "tokens_seen": 244187136 }, { "epoch": 0.07, "learning_rate": 0.0007482768304914745, "loss": 1.5676, "theoretical_loss": 4.277835764000752, "tokens_seen": 244252672 }, { "epoch": 0.07, "learning_rate": 0.0007482607823470411, "loss": 1.6488, "theoretical_loss": 4.277697517622459, "tokens_seen": 244318208 }, { "epoch": 0.07, "learning_rate": 0.0007482447342026079, "loss": 1.6331, "theoretical_loss": 4.277559318702588, "tokens_seen": 244383744 }, { "epoch": 0.07, "learning_rate": 0.0007482286860581746, "loss": 1.5206, "theoretical_loss": 4.277421167212125, "tokens_seen": 244449280 }, { "epoch": 0.07, "learning_rate": 0.0007482126379137413, "loss": 1.6105, "theoretical_loss": 4.277283063122083, "tokens_seen": 244514816 }, { "epoch": 0.07, "learning_rate": 0.000748196589769308, "loss": 1.6427, "theoretical_loss": 4.277145006403503, "tokens_seen": 244580352 }, { "epoch": 0.07, "learning_rate": 0.0007481805416248747, "loss": 1.7534, "theoretical_loss": 4.2770069970274465, "tokens_seen": 244645888 }, { "epoch": 0.07, "learning_rate": 0.0007481644934804414, "loss": 1.6926, "theoretical_loss": 4.276869034965005, "tokens_seen": 244711424 }, { "epoch": 0.07, "learning_rate": 0.000748148445336008, "loss": 1.7342, "theoretical_loss": 4.276731120187293, "tokens_seen": 244776960 }, { "epoch": 0.07, "learning_rate": 0.0007481323971915748, "loss": 1.6409, "theoretical_loss": 4.2765932526654495, "tokens_seen": 244842496 }, { "epoch": 0.07, "learning_rate": 0.0007481163490471414, "loss": 1.6483, "theoretical_loss": 4.276455432370643, "tokens_seen": 244908032 }, { "epoch": 0.07, "learning_rate": 0.0007481003009027082, "loss": 1.5419, "theoretical_loss": 4.276317659274062, "tokens_seen": 244973568 }, { "epoch": 0.07, "learning_rate": 0.0007480842527582748, "loss": 1.5604, "theoretical_loss": 4.276179933346924, "tokens_seen": 245039104 }, { "epoch": 0.07, "learning_rate": 0.0007480682046138416, "loss": 1.4418, "theoretical_loss": 4.27604225456047, "tokens_seen": 245104640 }, { "epoch": 0.07, "learning_rate": 0.0007480521564694082, "loss": 1.6223, "theoretical_loss": 4.275904622885967, "tokens_seen": 245170176 }, { "epoch": 0.07, "learning_rate": 0.000748036108324975, "loss": 1.6681, "theoretical_loss": 4.275767038294708, "tokens_seen": 245235712 }, { "epoch": 0.07, "learning_rate": 0.0007480200601805416, "loss": 1.4692, "theoretical_loss": 4.275629500758008, "tokens_seen": 245301248 }, { "epoch": 0.07, "learning_rate": 0.0007480040120361084, "loss": 1.5969, "theoretical_loss": 4.27549201024721, "tokens_seen": 245366784 }, { "epoch": 0.07, "learning_rate": 0.000747987963891675, "loss": 1.6522, "theoretical_loss": 4.275354566733682, "tokens_seen": 245432320 }, { "epoch": 0.07, "learning_rate": 0.0007479719157472418, "loss": 1.6379, "theoretical_loss": 4.2752171701888155, "tokens_seen": 245497856 }, { "epoch": 0.07, "learning_rate": 0.0007479558676028084, "loss": 1.5753, "theoretical_loss": 4.275079820584029, "tokens_seen": 245563392 }, { "epoch": 0.07, "learning_rate": 0.0007479398194583752, "loss": 1.6753, "theoretical_loss": 4.274942517890764, "tokens_seen": 245628928 }, { "epoch": 0.07, "learning_rate": 0.0007479237713139418, "loss": 1.6301, "theoretical_loss": 4.2748052620804895, "tokens_seen": 245694464 }, { "epoch": 0.07, "objective/train/docs_used": 87859, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.655349850654602, "objective/train/theoretical_loss": 4.274668053124696, "objective/train/tokens_used": 266220000, "theoretical_loss": 4.274668053124696, "tokens_seen": 245760000 }, { "epoch": 0.07, "learning_rate": 0.0007479077231695086, "loss": 1.6396, "theoretical_loss": 4.274668053124696, "tokens_seen": 245760000 }, { "epoch": 0.07, "learning_rate": 0.0007478916750250752, "loss": 1.5806, "theoretical_loss": 4.2745308909949005, "tokens_seen": 245825536 }, { "epoch": 0.07, "learning_rate": 0.000747875626880642, "loss": 1.7082, "theoretical_loss": 4.274393775662648, "tokens_seen": 245891072 }, { "epoch": 0.07, "learning_rate": 0.0007478595787362086, "loss": 1.6241, "theoretical_loss": 4.274256707099502, "tokens_seen": 245956608 }, { "epoch": 0.07, "learning_rate": 0.0007478435305917754, "loss": 1.5844, "theoretical_loss": 4.274119685277058, "tokens_seen": 246022144 }, { "epoch": 0.07, "learning_rate": 0.000747827482447342, "loss": 1.5921, "theoretical_loss": 4.273982710166931, "tokens_seen": 246087680 }, { "epoch": 0.07, "learning_rate": 0.0007478114343029088, "loss": 1.6682, "theoretical_loss": 4.273845781740762, "tokens_seen": 246153216 }, { "epoch": 0.07, "learning_rate": 0.0007477953861584754, "loss": 1.5675, "theoretical_loss": 4.273708899970218, "tokens_seen": 246218752 }, { "epoch": 0.07, "learning_rate": 0.0007477793380140422, "loss": 1.7248, "theoretical_loss": 4.273572064826991, "tokens_seen": 246284288 }, { "epoch": 0.07, "learning_rate": 0.0007477632898696088, "loss": 1.6493, "theoretical_loss": 4.273435276282795, "tokens_seen": 246349824 }, { "epoch": 0.07, "learning_rate": 0.0007477472417251756, "loss": 1.6431, "theoretical_loss": 4.2732985343093715, "tokens_seen": 246415360 }, { "epoch": 0.07, "learning_rate": 0.0007477311935807422, "loss": 1.5834, "theoretical_loss": 4.273161838878485, "tokens_seen": 246480896 }, { "epoch": 0.07, "learning_rate": 0.000747715145436309, "loss": 1.649, "theoretical_loss": 4.273025189961926, "tokens_seen": 246546432 }, { "epoch": 0.07, "learning_rate": 0.0007476990972918756, "loss": 1.5257, "theoretical_loss": 4.272888587531508, "tokens_seen": 246611968 }, { "epoch": 0.07, "learning_rate": 0.0007476830491474424, "loss": 1.7432, "theoretical_loss": 4.272752031559071, "tokens_seen": 246677504 }, { "epoch": 0.07, "learning_rate": 0.000747667001003009, "loss": 1.5807, "theoretical_loss": 4.272615522016476, "tokens_seen": 246743040 }, { "epoch": 0.07, "learning_rate": 0.0007476509528585758, "loss": 1.5903, "theoretical_loss": 4.272479058875614, "tokens_seen": 246808576 }, { "epoch": 0.07, "learning_rate": 0.0007476349047141424, "loss": 1.5591, "theoretical_loss": 4.272342642108396, "tokens_seen": 246874112 }, { "epoch": 0.07, "learning_rate": 0.0007476188565697092, "loss": 1.5049, "theoretical_loss": 4.272206271686758, "tokens_seen": 246939648 }, { "epoch": 0.07, "learning_rate": 0.0007476028084252758, "loss": 1.6674, "theoretical_loss": 4.272069947582662, "tokens_seen": 247005184 }, { "epoch": 0.07, "learning_rate": 0.0007475867602808426, "loss": 1.5916, "theoretical_loss": 4.271933669768094, "tokens_seen": 247070720 }, { "epoch": 0.07, "learning_rate": 0.0007475707121364092, "loss": 1.6944, "theoretical_loss": 4.271797438215064, "tokens_seen": 247136256 }, { "epoch": 0.07, "learning_rate": 0.000747554663991976, "loss": 1.5855, "theoretical_loss": 4.271661252895605, "tokens_seen": 247201792 }, { "epoch": 0.07, "learning_rate": 0.0007475386158475426, "loss": 1.7893, "theoretical_loss": 4.271525113781777, "tokens_seen": 247267328 }, { "epoch": 0.07, "learning_rate": 0.0007475225677031094, "loss": 1.6591, "theoretical_loss": 4.271389020845662, "tokens_seen": 247332864 }, { "epoch": 0.07, "learning_rate": 0.000747506519558676, "loss": 1.6706, "theoretical_loss": 4.27125297405937, "tokens_seen": 247398400 }, { "epoch": 0.07, "learning_rate": 0.0007474904714142428, "loss": 1.6419, "theoretical_loss": 4.271116973395028, "tokens_seen": 247463936 }, { "epoch": 0.08, "learning_rate": 0.0007474744232698094, "loss": 1.7433, "theoretical_loss": 4.270981018824795, "tokens_seen": 247529472 }, { "epoch": 0.08, "learning_rate": 0.0007474583751253762, "loss": 1.6313, "theoretical_loss": 4.270845110320851, "tokens_seen": 247595008 }, { "epoch": 0.08, "learning_rate": 0.0007474423269809428, "loss": 1.6649, "theoretical_loss": 4.270709247855397, "tokens_seen": 247660544 }, { "epoch": 0.08, "learning_rate": 0.0007474262788365096, "loss": 1.5666, "theoretical_loss": 4.270573431400663, "tokens_seen": 247726080 }, { "epoch": 0.08, "learning_rate": 0.0007474102306920762, "loss": 1.729, "theoretical_loss": 4.270437660928902, "tokens_seen": 247791616 }, { "epoch": 0.08, "learning_rate": 0.000747394182547643, "loss": 1.6638, "theoretical_loss": 4.270301936412389, "tokens_seen": 247857152 }, { "epoch": 0.08, "learning_rate": 0.0007473781344032096, "loss": 1.6416, "theoretical_loss": 4.2701662578234245, "tokens_seen": 247922688 }, { "epoch": 0.08, "learning_rate": 0.0007473620862587764, "loss": 1.5788, "theoretical_loss": 4.270030625134334, "tokens_seen": 247988224 }, { "epoch": 0.08, "learning_rate": 0.000747346038114343, "loss": 1.5882, "theoretical_loss": 4.269895038317464, "tokens_seen": 248053760 }, { "epoch": 0.08, "learning_rate": 0.0007473299899699098, "loss": 1.5104, "theoretical_loss": 4.269759497345188, "tokens_seen": 248119296 }, { "epoch": 0.08, "learning_rate": 0.0007473139418254764, "loss": 1.6717, "theoretical_loss": 4.269624002189901, "tokens_seen": 248184832 }, { "epoch": 0.08, "learning_rate": 0.0007472978936810432, "loss": 1.7941, "theoretical_loss": 4.269488552824024, "tokens_seen": 248250368 }, { "epoch": 0.08, "learning_rate": 0.0007472818455366098, "loss": 1.7017, "theoretical_loss": 4.269353149220001, "tokens_seen": 248315904 }, { "epoch": 0.08, "learning_rate": 0.0007472657973921766, "loss": 1.5691, "theoretical_loss": 4.2692177913503, "tokens_seen": 248381440 }, { "epoch": 0.08, "learning_rate": 0.0007472497492477432, "loss": 1.5977, "theoretical_loss": 4.2690824791874125, "tokens_seen": 248446976 }, { "epoch": 0.08, "learning_rate": 0.00074723370110331, "loss": 1.6137, "theoretical_loss": 4.2689472127038535, "tokens_seen": 248512512 }, { "epoch": 0.08, "learning_rate": 0.0007472176529588766, "loss": 1.5904, "theoretical_loss": 4.2688119918721625, "tokens_seen": 248578048 }, { "epoch": 0.08, "learning_rate": 0.0007472016048144434, "loss": 1.6154, "theoretical_loss": 4.268676816664902, "tokens_seen": 248643584 }, { "epoch": 0.08, "learning_rate": 0.00074718555667001, "loss": 1.6636, "theoretical_loss": 4.26854168705466, "tokens_seen": 248709120 }, { "epoch": 0.08, "learning_rate": 0.0007471695085255768, "loss": 1.6526, "theoretical_loss": 4.2684066030140455, "tokens_seen": 248774656 }, { "epoch": 0.08, "learning_rate": 0.0007471534603811434, "loss": 1.5885, "theoretical_loss": 4.268271564515693, "tokens_seen": 248840192 }, { "epoch": 0.08, "learning_rate": 0.0007471374122367102, "loss": 1.6694, "theoretical_loss": 4.26813657153226, "tokens_seen": 248905728 }, { "epoch": 0.08, "learning_rate": 0.000747121364092277, "loss": 1.6456, "theoretical_loss": 4.268001624036429, "tokens_seen": 248971264 }, { "epoch": 0.08, "objective/train/docs_used": 89113, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5617129802703857, "objective/train/theoretical_loss": 4.267866722000902, "objective/train/tokens_used": 269496800, "theoretical_loss": 4.267866722000902, "tokens_seen": 249036800 }, { "epoch": 0.08, "learning_rate": 0.0007471053159478436, "loss": 1.6179, "theoretical_loss": 4.267866722000902, "tokens_seen": 249036800 }, { "epoch": 0.08, "learning_rate": 0.0007470892678034103, "loss": 1.8281, "theoretical_loss": 4.26773186539841, "tokens_seen": 249102336 }, { "epoch": 0.08, "learning_rate": 0.000747073219658977, "loss": 1.6578, "theoretical_loss": 4.267597054201705, "tokens_seen": 249167872 }, { "epoch": 0.08, "learning_rate": 0.0007470571715145438, "loss": 1.7296, "theoretical_loss": 4.267462288383561, "tokens_seen": 249233408 }, { "epoch": 0.08, "learning_rate": 0.0007470411233701104, "loss": 1.5899, "theoretical_loss": 4.267327567916777, "tokens_seen": 249298944 }, { "epoch": 0.08, "learning_rate": 0.0007470250752256772, "loss": 1.6601, "theoretical_loss": 4.267192892774176, "tokens_seen": 249364480 }, { "epoch": 0.08, "learning_rate": 0.0007470090270812438, "loss": 1.5475, "theoretical_loss": 4.267058262928604, "tokens_seen": 249430016 }, { "epoch": 0.08, "learning_rate": 0.0007469929789368106, "loss": 1.6116, "theoretical_loss": 4.26692367835293, "tokens_seen": 249495552 }, { "epoch": 0.08, "learning_rate": 0.0007469769307923772, "loss": 1.4896, "theoretical_loss": 4.266789139020045, "tokens_seen": 249561088 }, { "epoch": 0.08, "learning_rate": 0.000746960882647944, "loss": 1.734, "theoretical_loss": 4.266654644902868, "tokens_seen": 249626624 }, { "epoch": 0.08, "learning_rate": 0.0007469448345035106, "loss": 1.6688, "theoretical_loss": 4.266520195974336, "tokens_seen": 249692160 }, { "epoch": 0.08, "learning_rate": 0.0007469287863590774, "loss": 1.6633, "theoretical_loss": 4.266385792207412, "tokens_seen": 249757696 }, { "epoch": 0.08, "learning_rate": 0.000746912738214644, "loss": 1.7313, "theoretical_loss": 4.266251433575082, "tokens_seen": 249823232 }, { "epoch": 0.08, "learning_rate": 0.0007468966900702106, "loss": 1.6623, "theoretical_loss": 4.266117120050355, "tokens_seen": 249888768 }, { "epoch": 0.08, "learning_rate": 0.0007468806419257774, "loss": 1.659, "theoretical_loss": 4.265982851606263, "tokens_seen": 249954304 }, { "epoch": 0.08, "learning_rate": 0.000746864593781344, "loss": 1.7441, "theoretical_loss": 4.265848628215862, "tokens_seen": 250019840 }, { "epoch": 0.08, "learning_rate": 0.0007468485456369108, "loss": 1.7166, "theoretical_loss": 4.265714449852229, "tokens_seen": 250085376 }, { "epoch": 0.08, "learning_rate": 0.0007468324974924775, "loss": 1.7004, "theoretical_loss": 4.265580316488467, "tokens_seen": 250150912 }, { "epoch": 0.08, "learning_rate": 0.0007468164493480442, "loss": 1.6103, "theoretical_loss": 4.2654462280977015, "tokens_seen": 250216448 }, { "epoch": 0.08, "learning_rate": 0.0007468004012036109, "loss": 1.5756, "theoretical_loss": 4.265312184653078, "tokens_seen": 250281984 }, { "epoch": 0.08, "learning_rate": 0.0007467843530591776, "loss": 1.6858, "theoretical_loss": 4.2651781861277716, "tokens_seen": 250347520 }, { "epoch": 0.08, "learning_rate": 0.0007467683049147443, "loss": 1.7365, "theoretical_loss": 4.265044232494972, "tokens_seen": 250413056 }, { "epoch": 0.08, "learning_rate": 0.0007467522567703109, "loss": 1.7981, "theoretical_loss": 4.264910323727898, "tokens_seen": 250478592 }, { "epoch": 0.08, "learning_rate": 0.0007467362086258777, "loss": 1.6383, "theoretical_loss": 4.264776459799791, "tokens_seen": 250544128 }, { "epoch": 0.08, "learning_rate": 0.0007467201604814443, "loss": 1.6194, "theoretical_loss": 4.264642640683911, "tokens_seen": 250609664 }, { "epoch": 0.08, "learning_rate": 0.0007467041123370111, "loss": 1.6848, "theoretical_loss": 4.264508866353546, "tokens_seen": 250675200 }, { "epoch": 0.08, "learning_rate": 0.0007466880641925777, "loss": 1.7286, "theoretical_loss": 4.264375136782004, "tokens_seen": 250740736 }, { "epoch": 0.08, "learning_rate": 0.0007466720160481445, "loss": 1.7089, "theoretical_loss": 4.264241451942617, "tokens_seen": 250806272 }, { "epoch": 0.08, "learning_rate": 0.0007466559679037111, "loss": 1.5735, "theoretical_loss": 4.264107811808739, "tokens_seen": 250871808 }, { "epoch": 0.08, "learning_rate": 0.0007466399197592779, "loss": 1.6118, "theoretical_loss": 4.263974216353749, "tokens_seen": 250937344 }, { "epoch": 0.08, "learning_rate": 0.0007466238716148445, "loss": 1.5047, "theoretical_loss": 4.2638406655510455, "tokens_seen": 251002880 }, { "epoch": 0.08, "learning_rate": 0.0007466078234704113, "loss": 1.6449, "theoretical_loss": 4.263707159374051, "tokens_seen": 251068416 }, { "epoch": 0.08, "learning_rate": 0.0007465917753259779, "loss": 1.5712, "theoretical_loss": 4.263573697796213, "tokens_seen": 251133952 }, { "epoch": 0.08, "learning_rate": 0.0007465757271815447, "loss": 1.6243, "theoretical_loss": 4.263440280790999, "tokens_seen": 251199488 }, { "epoch": 0.08, "learning_rate": 0.0007465596790371113, "loss": 1.6167, "theoretical_loss": 4.263306908331899, "tokens_seen": 251265024 }, { "epoch": 0.08, "learning_rate": 0.0007465436308926781, "loss": 1.6755, "theoretical_loss": 4.2631735803924276, "tokens_seen": 251330560 }, { "epoch": 0.08, "learning_rate": 0.0007465275827482447, "loss": 1.7118, "theoretical_loss": 4.263040296946122, "tokens_seen": 251396096 }, { "epoch": 0.08, "learning_rate": 0.0007465115346038115, "loss": 1.7401, "theoretical_loss": 4.26290705796654, "tokens_seen": 251461632 }, { "epoch": 0.08, "learning_rate": 0.0007464954864593781, "loss": 1.6834, "theoretical_loss": 4.262773863427265, "tokens_seen": 251527168 }, { "epoch": 0.08, "learning_rate": 0.0007464794383149449, "loss": 1.7491, "theoretical_loss": 4.262640713301899, "tokens_seen": 251592704 }, { "epoch": 0.08, "learning_rate": 0.0007464633901705115, "loss": 1.5683, "theoretical_loss": 4.26250760756407, "tokens_seen": 251658240 }, { "epoch": 0.08, "learning_rate": 0.0007464473420260783, "loss": 1.6373, "theoretical_loss": 4.262374546187428, "tokens_seen": 251723776 }, { "epoch": 0.08, "learning_rate": 0.0007464312938816449, "loss": 1.4961, "theoretical_loss": 4.262241529145643, "tokens_seen": 251789312 }, { "epoch": 0.08, "learning_rate": 0.0007464152457372117, "loss": 1.6324, "theoretical_loss": 4.262108556412411, "tokens_seen": 251854848 }, { "epoch": 0.08, "learning_rate": 0.0007463991975927783, "loss": 1.8033, "theoretical_loss": 4.261975627961448, "tokens_seen": 251920384 }, { "epoch": 0.08, "learning_rate": 0.0007463831494483451, "loss": 1.7477, "theoretical_loss": 4.2618427437664925, "tokens_seen": 251985920 }, { "epoch": 0.08, "learning_rate": 0.0007463671013039117, "loss": 1.6968, "theoretical_loss": 4.261709903801307, "tokens_seen": 252051456 }, { "epoch": 0.08, "learning_rate": 0.0007463510531594785, "loss": 1.6687, "theoretical_loss": 4.261577108039674, "tokens_seen": 252116992 }, { "epoch": 0.08, "learning_rate": 0.0007463350050150451, "loss": 1.7375, "theoretical_loss": 4.261444356455402, "tokens_seen": 252182528 }, { "epoch": 0.08, "learning_rate": 0.0007463189568706119, "loss": 1.6986, "theoretical_loss": 4.261311649022318, "tokens_seen": 252248064 }, { "epoch": 0.08, "objective/train/docs_used": 90341, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.549142599105835, "objective/train/theoretical_loss": 4.261178985714273, "objective/train/tokens_used": 272773600, "theoretical_loss": 4.261178985714273, "tokens_seen": 252313600 }, { "epoch": 0.08, "learning_rate": 0.0007463029087261785, "loss": 1.6768, "theoretical_loss": 4.261178985714273, "tokens_seen": 252313600 }, { "epoch": 0.08, "learning_rate": 0.0007462868605817453, "loss": 1.5756, "theoretical_loss": 4.26104636650514, "tokens_seen": 252379136 }, { "epoch": 0.08, "learning_rate": 0.0007462708124373119, "loss": 1.4787, "theoretical_loss": 4.260913791368815, "tokens_seen": 252444672 }, { "epoch": 0.08, "learning_rate": 0.0007462547642928787, "loss": 1.5706, "theoretical_loss": 4.260781260279216, "tokens_seen": 252510208 }, { "epoch": 0.08, "learning_rate": 0.0007462387161484453, "loss": 1.5784, "theoretical_loss": 4.260648773210281, "tokens_seen": 252575744 }, { "epoch": 0.08, "learning_rate": 0.0007462226680040121, "loss": 1.6752, "theoretical_loss": 4.2605163301359745, "tokens_seen": 252641280 }, { "epoch": 0.08, "learning_rate": 0.0007462066198595787, "loss": 1.5707, "theoretical_loss": 4.260383931030279, "tokens_seen": 252706816 }, { "epoch": 0.08, "learning_rate": 0.0007461905717151455, "loss": 1.6033, "theoretical_loss": 4.2602515758672, "tokens_seen": 252772352 }, { "epoch": 0.08, "learning_rate": 0.0007461745235707121, "loss": 1.7105, "theoretical_loss": 4.260119264620768, "tokens_seen": 252837888 }, { "epoch": 0.08, "learning_rate": 0.0007461584754262789, "loss": 1.7694, "theoretical_loss": 4.259986997265033, "tokens_seen": 252903424 }, { "epoch": 0.08, "learning_rate": 0.0007461424272818455, "loss": 1.5492, "theoretical_loss": 4.259854773774066, "tokens_seen": 252968960 }, { "epoch": 0.08, "learning_rate": 0.0007461263791374123, "loss": 1.5911, "theoretical_loss": 4.259722594121963, "tokens_seen": 253034496 }, { "epoch": 0.08, "learning_rate": 0.0007461103309929789, "loss": 1.6801, "theoretical_loss": 4.259590458282839, "tokens_seen": 253100032 }, { "epoch": 0.08, "learning_rate": 0.0007460942828485457, "loss": 1.6821, "theoretical_loss": 4.259458366230835, "tokens_seen": 253165568 }, { "epoch": 0.08, "learning_rate": 0.0007460782347041123, "loss": 1.6812, "theoretical_loss": 4.2593263179401095, "tokens_seen": 253231104 }, { "epoch": 0.08, "learning_rate": 0.0007460621865596791, "loss": 1.6885, "theoretical_loss": 4.259194313384846, "tokens_seen": 253296640 }, { "epoch": 0.08, "learning_rate": 0.0007460461384152457, "loss": 1.6347, "theoretical_loss": 4.259062352539247, "tokens_seen": 253362176 }, { "epoch": 0.08, "learning_rate": 0.0007460300902708125, "loss": 1.6389, "theoretical_loss": 4.258930435377541, "tokens_seen": 253427712 }, { "epoch": 0.08, "learning_rate": 0.0007460140421263791, "loss": 1.6788, "theoretical_loss": 4.258798561873974, "tokens_seen": 253493248 }, { "epoch": 0.08, "learning_rate": 0.0007459979939819459, "loss": 1.6139, "theoretical_loss": 4.258666732002817, "tokens_seen": 253558784 }, { "epoch": 0.08, "learning_rate": 0.0007459819458375125, "loss": 1.6973, "theoretical_loss": 4.258534945738362, "tokens_seen": 253624320 }, { "epoch": 0.08, "learning_rate": 0.0007459658976930793, "loss": 1.6564, "theoretical_loss": 4.258403203054923, "tokens_seen": 253689856 }, { "epoch": 0.08, "learning_rate": 0.0007459498495486459, "loss": 1.6421, "theoretical_loss": 4.258271503926833, "tokens_seen": 253755392 }, { "epoch": 0.08, "learning_rate": 0.0007459338014042127, "loss": 1.6334, "theoretical_loss": 4.258139848328451, "tokens_seen": 253820928 }, { "epoch": 0.08, "learning_rate": 0.0007459177532597794, "loss": 1.6213, "theoretical_loss": 4.258008236234154, "tokens_seen": 253886464 }, { "epoch": 0.08, "learning_rate": 0.0007459017051153461, "loss": 1.6186, "theoretical_loss": 4.257876667618344, "tokens_seen": 253952000 }, { "epoch": 0.08, "learning_rate": 0.0007458856569709128, "loss": 1.6503, "theoretical_loss": 4.257745142455442, "tokens_seen": 254017536 }, { "epoch": 0.08, "learning_rate": 0.0007458696088264795, "loss": 1.7115, "theoretical_loss": 4.257613660719892, "tokens_seen": 254083072 }, { "epoch": 0.08, "learning_rate": 0.0007458535606820462, "loss": 1.6601, "theoretical_loss": 4.25748222238616, "tokens_seen": 254148608 }, { "epoch": 0.08, "learning_rate": 0.0007458375125376129, "loss": 1.5442, "theoretical_loss": 4.2573508274287315, "tokens_seen": 254214144 }, { "epoch": 0.08, "learning_rate": 0.0007458214643931796, "loss": 1.4927, "theoretical_loss": 4.257219475822116, "tokens_seen": 254279680 }, { "epoch": 0.08, "learning_rate": 0.0007458054162487463, "loss": 1.5835, "theoretical_loss": 4.257088167540843, "tokens_seen": 254345216 }, { "epoch": 0.08, "learning_rate": 0.000745789368104313, "loss": 1.5372, "theoretical_loss": 4.256956902559465, "tokens_seen": 254410752 }, { "epoch": 0.08, "learning_rate": 0.0007457733199598797, "loss": 1.6738, "theoretical_loss": 4.256825680852554, "tokens_seen": 254476288 }, { "epoch": 0.08, "learning_rate": 0.0007457572718154464, "loss": 1.646, "theoretical_loss": 4.256694502394704, "tokens_seen": 254541824 }, { "epoch": 0.08, "learning_rate": 0.0007457412236710131, "loss": 1.6983, "theoretical_loss": 4.256563367160533, "tokens_seen": 254607360 }, { "epoch": 0.08, "learning_rate": 0.0007457251755265798, "loss": 1.6854, "theoretical_loss": 4.256432275124676, "tokens_seen": 254672896 }, { "epoch": 0.08, "learning_rate": 0.0007457091273821465, "loss": 1.6195, "theoretical_loss": 4.256301226261794, "tokens_seen": 254738432 }, { "epoch": 0.08, "learning_rate": 0.0007456930792377133, "loss": 1.5589, "theoretical_loss": 4.256170220546565, "tokens_seen": 254803968 }, { "epoch": 0.08, "learning_rate": 0.0007456770310932799, "loss": 1.4944, "theoretical_loss": 4.2560392579536925, "tokens_seen": 254869504 }, { "epoch": 0.08, "learning_rate": 0.0007456609829488467, "loss": 1.5076, "theoretical_loss": 4.255908338457898, "tokens_seen": 254935040 }, { "epoch": 0.08, "learning_rate": 0.0007456449348044133, "loss": 1.5108, "theoretical_loss": 4.255777462033926, "tokens_seen": 255000576 }, { "epoch": 0.08, "learning_rate": 0.00074562888665998, "loss": 1.5947, "theoretical_loss": 4.255646628656542, "tokens_seen": 255066112 }, { "epoch": 0.08, "learning_rate": 0.0007456128385155467, "loss": 1.5601, "theoretical_loss": 4.255515838300534, "tokens_seen": 255131648 }, { "epoch": 0.08, "learning_rate": 0.0007455967903711135, "loss": 1.6591, "theoretical_loss": 4.255385090940708, "tokens_seen": 255197184 }, { "epoch": 0.08, "learning_rate": 0.0007455807422266801, "loss": 1.6293, "theoretical_loss": 4.255254386551894, "tokens_seen": 255262720 }, { "epoch": 0.08, "learning_rate": 0.0007455646940822469, "loss": 1.5927, "theoretical_loss": 4.255123725108943, "tokens_seen": 255328256 }, { "epoch": 0.08, "learning_rate": 0.0007455486459378135, "loss": 1.584, "theoretical_loss": 4.254993106586725, "tokens_seen": 255393792 }, { "epoch": 0.08, "learning_rate": 0.0007455325977933803, "loss": 1.6033, "theoretical_loss": 4.254862530960134, "tokens_seen": 255459328 }, { "epoch": 0.08, "learning_rate": 0.0007455165496489469, "loss": 1.628, "theoretical_loss": 4.254731998204083, "tokens_seen": 255524864 }, { "epoch": 0.08, "objective/train/docs_used": 91319, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5541565418243408, "objective/train/theoretical_loss": 4.254601508293508, "objective/train/tokens_used": 276050400, "theoretical_loss": 4.254601508293508, "tokens_seen": 255590400 }, { "epoch": 0.08, "learning_rate": 0.0007455005015045136, "loss": 1.5715, "theoretical_loss": 4.254601508293508, "tokens_seen": 255590400 }, { "epoch": 0.08, "learning_rate": 0.0007454844533600803, "loss": 1.6553, "theoretical_loss": 4.254471061203364, "tokens_seen": 255655936 }, { "epoch": 0.08, "learning_rate": 0.000745468405215647, "loss": 1.7168, "theoretical_loss": 4.254340656908628, "tokens_seen": 255721472 }, { "epoch": 0.08, "learning_rate": 0.0007454523570712137, "loss": 1.636, "theoretical_loss": 4.254210295384298, "tokens_seen": 255787008 }, { "epoch": 0.08, "learning_rate": 0.0007454363089267804, "loss": 1.6813, "theoretical_loss": 4.254079976605394, "tokens_seen": 255852544 }, { "epoch": 0.08, "learning_rate": 0.0007454202607823471, "loss": 1.5497, "theoretical_loss": 4.253949700546956, "tokens_seen": 255918080 }, { "epoch": 0.08, "learning_rate": 0.0007454042126379138, "loss": 1.5719, "theoretical_loss": 4.2538194671840435, "tokens_seen": 255983616 }, { "epoch": 0.08, "learning_rate": 0.0007453881644934805, "loss": 1.7041, "theoretical_loss": 4.25368927649174, "tokens_seen": 256049152 }, { "epoch": 0.08, "learning_rate": 0.0007453721163490472, "loss": 1.5223, "theoretical_loss": 4.253559128445148, "tokens_seen": 256114688 }, { "epoch": 0.08, "learning_rate": 0.0007453560682046138, "loss": 1.6966, "theoretical_loss": 4.253429023019391, "tokens_seen": 256180224 }, { "epoch": 0.08, "learning_rate": 0.0007453400200601806, "loss": 1.6931, "theoretical_loss": 4.253298960189614, "tokens_seen": 256245760 }, { "epoch": 0.08, "learning_rate": 0.0007453239719157472, "loss": 1.5582, "theoretical_loss": 4.253168939930982, "tokens_seen": 256311296 }, { "epoch": 0.08, "learning_rate": 0.000745307923771314, "loss": 1.5669, "theoretical_loss": 4.253038962218682, "tokens_seen": 256376832 }, { "epoch": 0.08, "learning_rate": 0.0007452918756268806, "loss": 1.7125, "theoretical_loss": 4.25290902702792, "tokens_seen": 256442368 }, { "epoch": 0.08, "learning_rate": 0.0007452758274824474, "loss": 1.6, "theoretical_loss": 4.252779134333926, "tokens_seen": 256507904 }, { "epoch": 0.08, "learning_rate": 0.000745259779338014, "loss": 1.7343, "theoretical_loss": 4.252649284111946, "tokens_seen": 256573440 }, { "epoch": 0.08, "learning_rate": 0.0007452437311935808, "loss": 1.6937, "theoretical_loss": 4.252519476337251, "tokens_seen": 256638976 }, { "epoch": 0.08, "learning_rate": 0.0007452276830491474, "loss": 1.5897, "theoretical_loss": 4.252389710985131, "tokens_seen": 256704512 }, { "epoch": 0.08, "learning_rate": 0.0007452116349047142, "loss": 1.7181, "theoretical_loss": 4.252259988030896, "tokens_seen": 256770048 }, { "epoch": 0.08, "learning_rate": 0.0007451955867602808, "loss": 1.6677, "theoretical_loss": 4.252130307449878, "tokens_seen": 256835584 }, { "epoch": 0.08, "learning_rate": 0.0007451795386158476, "loss": 1.6119, "theoretical_loss": 4.252000669217429, "tokens_seen": 256901120 }, { "epoch": 0.08, "learning_rate": 0.0007451634904714142, "loss": 1.722, "theoretical_loss": 4.251871073308923, "tokens_seen": 256966656 }, { "epoch": 0.08, "learning_rate": 0.000745147442326981, "loss": 1.6881, "theoretical_loss": 4.251741519699751, "tokens_seen": 257032192 }, { "epoch": 0.08, "learning_rate": 0.0007451313941825476, "loss": 1.6454, "theoretical_loss": 4.251612008365328, "tokens_seen": 257097728 }, { "epoch": 0.08, "learning_rate": 0.0007451153460381144, "loss": 1.527, "theoretical_loss": 4.2514825392810875, "tokens_seen": 257163264 }, { "epoch": 0.08, "learning_rate": 0.000745099297893681, "loss": 1.6586, "theoretical_loss": 4.251353112422486, "tokens_seen": 257228800 }, { "epoch": 0.08, "learning_rate": 0.0007450832497492478, "loss": 1.6275, "theoretical_loss": 4.251223727764999, "tokens_seen": 257294336 }, { "epoch": 0.08, "learning_rate": 0.0007450672016048144, "loss": 1.716, "theoretical_loss": 4.251094385284121, "tokens_seen": 257359872 }, { "epoch": 0.08, "learning_rate": 0.0007450511534603812, "loss": 1.6393, "theoretical_loss": 4.25096508495537, "tokens_seen": 257425408 }, { "epoch": 0.08, "learning_rate": 0.0007450351053159478, "loss": 1.6811, "theoretical_loss": 4.250835826754281, "tokens_seen": 257490944 }, { "epoch": 0.08, "learning_rate": 0.0007450190571715146, "loss": 1.6071, "theoretical_loss": 4.2507066106564135, "tokens_seen": 257556480 }, { "epoch": 0.08, "learning_rate": 0.0007450030090270812, "loss": 1.7105, "theoretical_loss": 4.250577436637344, "tokens_seen": 257622016 }, { "epoch": 0.08, "learning_rate": 0.000744986960882648, "loss": 1.6548, "theoretical_loss": 4.25044830467267, "tokens_seen": 257687552 }, { "epoch": 0.08, "learning_rate": 0.0007449709127382146, "loss": 1.7264, "theoretical_loss": 4.250319214738011, "tokens_seen": 257753088 }, { "epoch": 0.08, "learning_rate": 0.0007449548645937814, "loss": 1.6103, "theoretical_loss": 4.250190166809005, "tokens_seen": 257818624 }, { "epoch": 0.08, "learning_rate": 0.000744938816449348, "loss": 1.6217, "theoretical_loss": 4.250061160861311, "tokens_seen": 257884160 }, { "epoch": 0.08, "learning_rate": 0.0007449227683049148, "loss": 1.5565, "theoretical_loss": 4.249932196870608, "tokens_seen": 257949696 }, { "epoch": 0.08, "learning_rate": 0.0007449067201604814, "loss": 1.6584, "theoretical_loss": 4.249803274812598, "tokens_seen": 258015232 }, { "epoch": 0.08, "learning_rate": 0.0007448906720160482, "loss": 1.7406, "theoretical_loss": 4.249674394662998, "tokens_seen": 258080768 }, { "epoch": 0.08, "learning_rate": 0.0007448746238716148, "loss": 1.5355, "theoretical_loss": 4.249545556397549, "tokens_seen": 258146304 }, { "epoch": 0.08, "learning_rate": 0.0007448585757271816, "loss": 1.6406, "theoretical_loss": 4.249416759992011, "tokens_seen": 258211840 }, { "epoch": 0.08, "learning_rate": 0.0007448425275827482, "loss": 1.5055, "theoretical_loss": 4.249288005422166, "tokens_seen": 258277376 }, { "epoch": 0.08, "learning_rate": 0.000744826479438315, "loss": 1.5312, "theoretical_loss": 4.249159292663812, "tokens_seen": 258342912 }, { "epoch": 0.08, "learning_rate": 0.0007448104312938816, "loss": 1.6441, "theoretical_loss": 4.249030621692772, "tokens_seen": 258408448 }, { "epoch": 0.08, "learning_rate": 0.0007447943831494484, "loss": 1.6091, "theoretical_loss": 4.248901992484885, "tokens_seen": 258473984 }, { "epoch": 0.08, "learning_rate": 0.000744778335005015, "loss": 1.6485, "theoretical_loss": 4.248773405016014, "tokens_seen": 258539520 }, { "epoch": 0.08, "learning_rate": 0.0007447622868605818, "loss": 1.6405, "theoretical_loss": 4.248644859262039, "tokens_seen": 258605056 }, { "epoch": 0.08, "learning_rate": 0.0007447462387161484, "loss": 1.5547, "theoretical_loss": 4.248516355198861, "tokens_seen": 258670592 }, { "epoch": 0.08, "learning_rate": 0.0007447301905717152, "loss": 1.6182, "theoretical_loss": 4.2483878928024, "tokens_seen": 258736128 }, { "epoch": 0.08, "learning_rate": 0.0007447141424272819, "loss": 1.592, "theoretical_loss": 4.248259472048598, "tokens_seen": 258801664 }, { "epoch": 0.08, "objective/train/docs_used": 92500, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5000085830688477, "objective/train/theoretical_loss": 4.248131092913416, "objective/train/tokens_used": 279327200, "theoretical_loss": 4.248131092913416, "tokens_seen": 258867200 }, { "epoch": 0.08, "learning_rate": 0.0007446980942828486, "loss": 1.5452, "theoretical_loss": 4.248131092913416, "tokens_seen": 258867200 }, { "epoch": 0.08, "learning_rate": 0.0007446820461384153, "loss": 1.542, "theoretical_loss": 4.248002755372836, "tokens_seen": 258932736 }, { "epoch": 0.08, "learning_rate": 0.000744665997993982, "loss": 1.6977, "theoretical_loss": 4.247874459402857, "tokens_seen": 258998272 }, { "epoch": 0.08, "learning_rate": 0.0007446499498495487, "loss": 1.5523, "theoretical_loss": 4.2477462049795, "tokens_seen": 259063808 }, { "epoch": 0.08, "learning_rate": 0.0007446339017051154, "loss": 1.6554, "theoretical_loss": 4.2476179920788075, "tokens_seen": 259129344 }, { "epoch": 0.08, "learning_rate": 0.0007446178535606821, "loss": 1.557, "theoretical_loss": 4.247489820676839, "tokens_seen": 259194880 }, { "epoch": 0.08, "learning_rate": 0.0007446018054162488, "loss": 1.6801, "theoretical_loss": 4.247361690749676, "tokens_seen": 259260416 }, { "epoch": 0.08, "learning_rate": 0.0007445857572718155, "loss": 1.604, "theoretical_loss": 4.2472336022734165, "tokens_seen": 259325952 }, { "epoch": 0.08, "learning_rate": 0.0007445697091273822, "loss": 1.6637, "theoretical_loss": 4.2471055552241825, "tokens_seen": 259391488 }, { "epoch": 0.08, "learning_rate": 0.0007445536609829489, "loss": 1.6168, "theoretical_loss": 4.246977549578113, "tokens_seen": 259457024 }, { "epoch": 0.08, "learning_rate": 0.0007445376128385156, "loss": 1.6913, "theoretical_loss": 4.246849585311368, "tokens_seen": 259522560 }, { "epoch": 0.08, "learning_rate": 0.0007445215646940823, "loss": 1.6747, "theoretical_loss": 4.246721662400128, "tokens_seen": 259588096 }, { "epoch": 0.08, "learning_rate": 0.000744505516549649, "loss": 1.5241, "theoretical_loss": 4.246593780820589, "tokens_seen": 259653632 }, { "epoch": 0.08, "learning_rate": 0.0007444894684052157, "loss": 1.5544, "theoretical_loss": 4.2464659405489735, "tokens_seen": 259719168 }, { "epoch": 0.08, "learning_rate": 0.0007444734202607824, "loss": 1.4963, "theoretical_loss": 4.246338141561518, "tokens_seen": 259784704 }, { "epoch": 0.08, "learning_rate": 0.0007444573721163491, "loss": 1.5923, "theoretical_loss": 4.24621038383448, "tokens_seen": 259850240 }, { "epoch": 0.08, "learning_rate": 0.0007444413239719158, "loss": 1.643, "theoretical_loss": 4.246082667344139, "tokens_seen": 259915776 }, { "epoch": 0.08, "learning_rate": 0.0007444252758274825, "loss": 1.5979, "theoretical_loss": 4.245954992066792, "tokens_seen": 259981312 }, { "epoch": 0.08, "learning_rate": 0.0007444092276830492, "loss": 1.7024, "theoretical_loss": 4.245827357978754, "tokens_seen": 260046848 }, { "epoch": 0.08, "learning_rate": 0.000744393179538616, "loss": 1.5572, "theoretical_loss": 4.245699765056365, "tokens_seen": 260112384 }, { "epoch": 0.08, "learning_rate": 0.0007443771313941826, "loss": 1.5896, "theoretical_loss": 4.245572213275978, "tokens_seen": 260177920 }, { "epoch": 0.08, "learning_rate": 0.0007443610832497493, "loss": 1.8514, "theoretical_loss": 4.245444702613971, "tokens_seen": 260243456 }, { "epoch": 0.08, "learning_rate": 0.000744345035105316, "loss": 1.586, "theoretical_loss": 4.245317233046738, "tokens_seen": 260308992 }, { "epoch": 0.08, "learning_rate": 0.0007443289869608828, "loss": 1.7146, "theoretical_loss": 4.2451898045506935, "tokens_seen": 260374528 }, { "epoch": 0.08, "learning_rate": 0.0007443129388164494, "loss": 1.6225, "theoretical_loss": 4.245062417102272, "tokens_seen": 260440064 }, { "epoch": 0.08, "learning_rate": 0.0007442968906720162, "loss": 1.592, "theoretical_loss": 4.244935070677927, "tokens_seen": 260505600 }, { "epoch": 0.08, "learning_rate": 0.0007442808425275828, "loss": 1.6766, "theoretical_loss": 4.244807765254132, "tokens_seen": 260571136 }, { "epoch": 0.08, "learning_rate": 0.0007442647943831496, "loss": 1.6643, "theoretical_loss": 4.244680500807378, "tokens_seen": 260636672 }, { "epoch": 0.08, "learning_rate": 0.0007442487462387162, "loss": 1.5448, "theoretical_loss": 4.244553277314178, "tokens_seen": 260702208 }, { "epoch": 0.08, "learning_rate": 0.000744232698094283, "loss": 1.6036, "theoretical_loss": 4.244426094751063, "tokens_seen": 260767744 }, { "epoch": 0.08, "learning_rate": 0.0007442166499498496, "loss": 1.6473, "theoretical_loss": 4.244298953094583, "tokens_seen": 260833280 }, { "epoch": 0.08, "learning_rate": 0.0007442006018054164, "loss": 1.6384, "theoretical_loss": 4.244171852321308, "tokens_seen": 260898816 }, { "epoch": 0.08, "learning_rate": 0.000744184553660983, "loss": 1.6358, "theoretical_loss": 4.244044792407828, "tokens_seen": 260964352 }, { "epoch": 0.08, "learning_rate": 0.0007441685055165498, "loss": 1.6318, "theoretical_loss": 4.24391777333075, "tokens_seen": 261029888 }, { "epoch": 0.08, "learning_rate": 0.0007441524573721164, "loss": 1.6431, "theoretical_loss": 4.243790795066703, "tokens_seen": 261095424 }, { "epoch": 0.08, "learning_rate": 0.0007441364092276832, "loss": 1.5781, "theoretical_loss": 4.243663857592332, "tokens_seen": 261160960 }, { "epoch": 0.08, "learning_rate": 0.0007441203610832498, "loss": 1.5393, "theoretical_loss": 4.243536960884305, "tokens_seen": 261226496 }, { "epoch": 0.08, "learning_rate": 0.0007441043129388166, "loss": 1.6555, "theoretical_loss": 4.243410104919307, "tokens_seen": 261292032 }, { "epoch": 0.08, "learning_rate": 0.0007440882647943832, "loss": 1.6852, "theoretical_loss": 4.243283289674042, "tokens_seen": 261357568 }, { "epoch": 0.08, "learning_rate": 0.0007440722166499499, "loss": 1.4922, "theoretical_loss": 4.243156515125234, "tokens_seen": 261423104 }, { "epoch": 0.08, "learning_rate": 0.0007440561685055166, "loss": 1.6475, "theoretical_loss": 4.243029781249625, "tokens_seen": 261488640 }, { "epoch": 0.08, "learning_rate": 0.0007440401203610833, "loss": 1.5878, "theoretical_loss": 4.2429030880239775, "tokens_seen": 261554176 }, { "epoch": 0.08, "learning_rate": 0.00074402407221665, "loss": 1.6059, "theoretical_loss": 4.242776435425072, "tokens_seen": 261619712 }, { "epoch": 0.08, "learning_rate": 0.0007440080240722167, "loss": 1.5873, "theoretical_loss": 4.24264982342971, "tokens_seen": 261685248 }, { "epoch": 0.08, "learning_rate": 0.0007439919759277834, "loss": 1.5429, "theoretical_loss": 4.242523252014709, "tokens_seen": 261750784 }, { "epoch": 0.08, "learning_rate": 0.0007439759277833501, "loss": 1.6651, "theoretical_loss": 4.242396721156908, "tokens_seen": 261816320 }, { "epoch": 0.08, "learning_rate": 0.0007439598796389168, "loss": 1.7161, "theoretical_loss": 4.242270230833164, "tokens_seen": 261881856 }, { "epoch": 0.08, "learning_rate": 0.0007439438314944835, "loss": 1.5095, "theoretical_loss": 4.242143781020353, "tokens_seen": 261947392 }, { "epoch": 0.08, "learning_rate": 0.0007439277833500501, "loss": 1.6434, "theoretical_loss": 4.24201737169537, "tokens_seen": 262012928 }, { "epoch": 0.08, "learning_rate": 0.0007439117352056169, "loss": 1.6189, "theoretical_loss": 4.24189100283513, "tokens_seen": 262078464 }, { "epoch": 0.08, "objective/train/docs_used": 93423, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6364566087722778, "objective/train/theoretical_loss": 4.241764674416565, "objective/train/tokens_used": 282604000, "theoretical_loss": 4.241764674416565, "tokens_seen": 262144000 }, { "epoch": 0.08, "learning_rate": 0.0007438956870611835, "loss": 1.6476, "theoretical_loss": 4.241764674416565, "tokens_seen": 262144000 }, { "epoch": 0.08, "learning_rate": 0.0007438796389167503, "loss": 1.5735, "theoretical_loss": 4.241638386416628, "tokens_seen": 262209536 }, { "epoch": 0.08, "learning_rate": 0.0007438635907723169, "loss": 1.6634, "theoretical_loss": 4.241512138812288, "tokens_seen": 262275072 }, { "epoch": 0.08, "learning_rate": 0.0007438475426278837, "loss": 1.5158, "theoretical_loss": 4.2413859315805365, "tokens_seen": 262340608 }, { "epoch": 0.08, "learning_rate": 0.0007438314944834503, "loss": 1.5613, "theoretical_loss": 4.241259764698381, "tokens_seen": 262406144 }, { "epoch": 0.08, "learning_rate": 0.0007438154463390171, "loss": 1.5698, "theoretical_loss": 4.24113363814285, "tokens_seen": 262471680 }, { "epoch": 0.08, "learning_rate": 0.0007437993981945837, "loss": 1.7, "theoretical_loss": 4.241007551890988, "tokens_seen": 262537216 }, { "epoch": 0.08, "learning_rate": 0.0007437833500501505, "loss": 1.5795, "theoretical_loss": 4.2408815059198615, "tokens_seen": 262602752 }, { "epoch": 0.08, "learning_rate": 0.0007437673019057171, "loss": 1.6302, "theoretical_loss": 4.240755500206554, "tokens_seen": 262668288 }, { "epoch": 0.08, "learning_rate": 0.0007437512537612839, "loss": 1.6311, "theoretical_loss": 4.240629534728167, "tokens_seen": 262733824 }, { "epoch": 0.08, "learning_rate": 0.0007437352056168505, "loss": 1.6255, "theoretical_loss": 4.240503609461822, "tokens_seen": 262799360 }, { "epoch": 0.08, "learning_rate": 0.0007437191574724173, "loss": 1.8019, "theoretical_loss": 4.24037772438466, "tokens_seen": 262864896 }, { "epoch": 0.08, "learning_rate": 0.0007437031093279839, "loss": 1.5956, "theoretical_loss": 4.240251879473839, "tokens_seen": 262930432 }, { "epoch": 0.08, "learning_rate": 0.0007436870611835507, "loss": 1.6039, "theoretical_loss": 4.240126074706536, "tokens_seen": 262995968 }, { "epoch": 0.08, "learning_rate": 0.0007436710130391173, "loss": 1.6342, "theoretical_loss": 4.240000310059948, "tokens_seen": 263061504 }, { "epoch": 0.08, "learning_rate": 0.0007436549648946841, "loss": 1.5717, "theoretical_loss": 4.239874585511288, "tokens_seen": 263127040 }, { "epoch": 0.08, "learning_rate": 0.0007436389167502507, "loss": 1.7075, "theoretical_loss": 4.239748901037791, "tokens_seen": 263192576 }, { "epoch": 0.08, "learning_rate": 0.0007436228686058175, "loss": 1.5863, "theoretical_loss": 4.2396232566167065, "tokens_seen": 263258112 }, { "epoch": 0.08, "learning_rate": 0.0007436068204613841, "loss": 1.7253, "theoretical_loss": 4.239497652225307, "tokens_seen": 263323648 }, { "epoch": 0.08, "learning_rate": 0.0007435907723169509, "loss": 1.5759, "theoretical_loss": 4.23937208784088, "tokens_seen": 263389184 }, { "epoch": 0.08, "learning_rate": 0.0007435747241725175, "loss": 1.5407, "theoretical_loss": 4.239246563440734, "tokens_seen": 263454720 }, { "epoch": 0.08, "learning_rate": 0.0007435586760280843, "loss": 1.512, "theoretical_loss": 4.239121079002194, "tokens_seen": 263520256 }, { "epoch": 0.08, "learning_rate": 0.000743542627883651, "loss": 1.5218, "theoretical_loss": 4.238995634502606, "tokens_seen": 263585792 }, { "epoch": 0.08, "learning_rate": 0.0007435265797392177, "loss": 1.6162, "theoretical_loss": 4.238870229919331, "tokens_seen": 263651328 }, { "epoch": 0.08, "learning_rate": 0.0007435105315947844, "loss": 1.5465, "theoretical_loss": 4.2387448652297515, "tokens_seen": 263716864 }, { "epoch": 0.08, "learning_rate": 0.0007434944834503511, "loss": 1.6953, "theoretical_loss": 4.238619540411268, "tokens_seen": 263782400 }, { "epoch": 0.08, "learning_rate": 0.0007434784353059178, "loss": 1.6432, "theoretical_loss": 4.238494255441296, "tokens_seen": 263847936 }, { "epoch": 0.08, "learning_rate": 0.0007434623871614845, "loss": 1.6252, "theoretical_loss": 4.238369010297275, "tokens_seen": 263913472 }, { "epoch": 0.08, "learning_rate": 0.0007434463390170512, "loss": 1.6152, "theoretical_loss": 4.238243804956658, "tokens_seen": 263979008 }, { "epoch": 0.08, "learning_rate": 0.0007434302908726179, "loss": 1.6624, "theoretical_loss": 4.23811863939692, "tokens_seen": 264044544 }, { "epoch": 0.08, "learning_rate": 0.0007434142427281846, "loss": 1.605, "theoretical_loss": 4.237993513595552, "tokens_seen": 264110080 }, { "epoch": 0.08, "learning_rate": 0.0007433981945837513, "loss": 1.5313, "theoretical_loss": 4.2378684275300635, "tokens_seen": 264175616 }, { "epoch": 0.08, "learning_rate": 0.000743382146439318, "loss": 1.5915, "theoretical_loss": 4.237743381177983, "tokens_seen": 264241152 }, { "epoch": 0.08, "learning_rate": 0.0007433660982948847, "loss": 1.6108, "theoretical_loss": 4.237618374516858, "tokens_seen": 264306688 }, { "epoch": 0.08, "learning_rate": 0.0007433500501504514, "loss": 1.6528, "theoretical_loss": 4.237493407524253, "tokens_seen": 264372224 }, { "epoch": 0.08, "learning_rate": 0.0007433340020060181, "loss": 1.5117, "theoretical_loss": 4.23736848017775, "tokens_seen": 264437760 }, { "epoch": 0.08, "learning_rate": 0.0007433179538615848, "loss": 1.6246, "theoretical_loss": 4.237243592454951, "tokens_seen": 264503296 }, { "epoch": 0.08, "learning_rate": 0.0007433019057171515, "loss": 1.6768, "theoretical_loss": 4.2371187443334755, "tokens_seen": 264568832 }, { "epoch": 0.08, "learning_rate": 0.0007432858575727182, "loss": 1.7698, "theoretical_loss": 4.236993935790962, "tokens_seen": 264634368 }, { "epoch": 0.08, "learning_rate": 0.0007432698094282849, "loss": 1.5859, "theoretical_loss": 4.236869166805064, "tokens_seen": 264699904 }, { "epoch": 0.08, "learning_rate": 0.0007432537612838516, "loss": 1.4231, "theoretical_loss": 4.236744437353458, "tokens_seen": 264765440 }, { "epoch": 0.08, "learning_rate": 0.0007432377131394183, "loss": 1.7275, "theoretical_loss": 4.236619747413836, "tokens_seen": 264830976 }, { "epoch": 0.08, "learning_rate": 0.000743221664994985, "loss": 1.5382, "theoretical_loss": 4.236495096963906, "tokens_seen": 264896512 }, { "epoch": 0.08, "learning_rate": 0.0007432056168505517, "loss": 1.6893, "theoretical_loss": 4.236370485981398, "tokens_seen": 264962048 }, { "epoch": 0.08, "learning_rate": 0.0007431895687061184, "loss": 1.676, "theoretical_loss": 4.236245914444058, "tokens_seen": 265027584 }, { "epoch": 0.08, "learning_rate": 0.0007431735205616851, "loss": 1.5495, "theoretical_loss": 4.2361213823296495, "tokens_seen": 265093120 }, { "epoch": 0.08, "learning_rate": 0.0007431574724172518, "loss": 1.6253, "theoretical_loss": 4.235996889615957, "tokens_seen": 265158656 }, { "epoch": 0.08, "learning_rate": 0.0007431414242728185, "loss": 1.646, "theoretical_loss": 4.235872436280779, "tokens_seen": 265224192 }, { "epoch": 0.08, "learning_rate": 0.0007431253761283852, "loss": 1.7312, "theoretical_loss": 4.235748022301935, "tokens_seen": 265289728 }, { "epoch": 0.08, "learning_rate": 0.0007431093279839519, "loss": 1.5934, "theoretical_loss": 4.235623647657261, "tokens_seen": 265355264 }, { "epoch": 0.08, "objective/train/docs_used": 94744, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6758512258529663, "objective/train/theoretical_loss": 4.235499312324611, "objective/train/tokens_used": 285880800, "theoretical_loss": 4.235499312324611, "tokens_seen": 265420800 }, { "epoch": 0.08, "learning_rate": 0.0007430932798395186, "loss": 1.6044, "theoretical_loss": 4.235499312324611, "tokens_seen": 265420800 }, { "epoch": 0.08, "learning_rate": 0.0007430772316950853, "loss": 1.6172, "theoretical_loss": 4.235375016281858, "tokens_seen": 265486336 }, { "epoch": 0.08, "learning_rate": 0.000743061183550652, "loss": 1.5535, "theoretical_loss": 4.235250759506892, "tokens_seen": 265551872 }, { "epoch": 0.08, "learning_rate": 0.0007430451354062187, "loss": 1.4944, "theoretical_loss": 4.235126541977621, "tokens_seen": 265617408 }, { "epoch": 0.08, "learning_rate": 0.0007430290872617854, "loss": 1.6137, "theoretical_loss": 4.2350023636719705, "tokens_seen": 265682944 }, { "epoch": 0.08, "learning_rate": 0.0007430130391173521, "loss": 1.4967, "theoretical_loss": 4.234878224567885, "tokens_seen": 265748480 }, { "epoch": 0.08, "learning_rate": 0.0007429969909729188, "loss": 1.5846, "theoretical_loss": 4.2347541246433265, "tokens_seen": 265814016 }, { "epoch": 0.08, "learning_rate": 0.0007429809428284855, "loss": 1.6215, "theoretical_loss": 4.234630063876273, "tokens_seen": 265879552 }, { "epoch": 0.08, "learning_rate": 0.0007429648946840523, "loss": 1.5888, "theoretical_loss": 4.234506042244723, "tokens_seen": 265945088 }, { "epoch": 0.08, "learning_rate": 0.0007429488465396189, "loss": 1.6198, "theoretical_loss": 4.234382059726691, "tokens_seen": 266010624 }, { "epoch": 0.08, "learning_rate": 0.0007429327983951857, "loss": 1.635, "theoretical_loss": 4.234258116300211, "tokens_seen": 266076160 }, { "epoch": 0.08, "learning_rate": 0.0007429167502507523, "loss": 1.5904, "theoretical_loss": 4.234134211943332, "tokens_seen": 266141696 }, { "epoch": 0.08, "learning_rate": 0.000742900702106319, "loss": 1.6317, "theoretical_loss": 4.234010346634123, "tokens_seen": 266207232 }, { "epoch": 0.08, "learning_rate": 0.0007428846539618857, "loss": 1.6059, "theoretical_loss": 4.23388652035067, "tokens_seen": 266272768 }, { "epoch": 0.08, "learning_rate": 0.0007428686058174525, "loss": 1.5262, "theoretical_loss": 4.233762733071076, "tokens_seen": 266338304 }, { "epoch": 0.08, "learning_rate": 0.0007428525576730191, "loss": 1.6322, "theoretical_loss": 4.233638984773464, "tokens_seen": 266403840 }, { "epoch": 0.08, "learning_rate": 0.0007428365095285859, "loss": 1.7282, "theoretical_loss": 4.2335152754359715, "tokens_seen": 266469376 }, { "epoch": 0.08, "learning_rate": 0.0007428204613841525, "loss": 1.5823, "theoretical_loss": 4.233391605036756, "tokens_seen": 266534912 }, { "epoch": 0.08, "learning_rate": 0.0007428044132397193, "loss": 1.555, "theoretical_loss": 4.233267973553991, "tokens_seen": 266600448 }, { "epoch": 0.08, "learning_rate": 0.0007427883650952859, "loss": 1.6832, "theoretical_loss": 4.233144380965869, "tokens_seen": 266665984 }, { "epoch": 0.08, "learning_rate": 0.0007427723169508527, "loss": 1.5752, "theoretical_loss": 4.2330208272505985, "tokens_seen": 266731520 }, { "epoch": 0.08, "learning_rate": 0.0007427562688064193, "loss": 1.6642, "theoretical_loss": 4.232897312386408, "tokens_seen": 266797056 }, { "epoch": 0.08, "learning_rate": 0.0007427402206619861, "loss": 1.6123, "theoretical_loss": 4.232773836351541, "tokens_seen": 266862592 }, { "epoch": 0.08, "learning_rate": 0.0007427241725175527, "loss": 1.6768, "theoretical_loss": 4.2326503991242586, "tokens_seen": 266928128 }, { "epoch": 0.08, "learning_rate": 0.0007427081243731195, "loss": 1.5605, "theoretical_loss": 4.232527000682843, "tokens_seen": 266993664 }, { "epoch": 0.08, "learning_rate": 0.0007426920762286861, "loss": 1.6535, "theoretical_loss": 4.232403641005589, "tokens_seen": 267059200 }, { "epoch": 0.08, "learning_rate": 0.0007426760280842528, "loss": 1.6531, "theoretical_loss": 4.232280320070812, "tokens_seen": 267124736 }, { "epoch": 0.08, "learning_rate": 0.0007426599799398195, "loss": 1.6343, "theoretical_loss": 4.232157037856844, "tokens_seen": 267190272 }, { "epoch": 0.08, "learning_rate": 0.0007426439317953862, "loss": 1.6088, "theoretical_loss": 4.232033794342033, "tokens_seen": 267255808 }, { "epoch": 0.08, "learning_rate": 0.0007426278836509529, "loss": 1.5571, "theoretical_loss": 4.231910589504748, "tokens_seen": 267321344 }, { "epoch": 0.08, "learning_rate": 0.0007426118355065196, "loss": 1.626, "theoretical_loss": 4.2317874233233725, "tokens_seen": 267386880 }, { "epoch": 0.08, "learning_rate": 0.0007425957873620863, "loss": 1.561, "theoretical_loss": 4.231664295776307, "tokens_seen": 267452416 }, { "epoch": 0.08, "learning_rate": 0.000742579739217653, "loss": 1.5373, "theoretical_loss": 4.231541206841971, "tokens_seen": 267517952 }, { "epoch": 0.08, "learning_rate": 0.0007425636910732197, "loss": 1.5562, "theoretical_loss": 4.2314181564988, "tokens_seen": 267583488 }, { "epoch": 0.08, "learning_rate": 0.0007425476429287864, "loss": 1.614, "theoretical_loss": 4.23129514472525, "tokens_seen": 267649024 }, { "epoch": 0.08, "learning_rate": 0.000742531594784353, "loss": 1.5684, "theoretical_loss": 4.231172171499789, "tokens_seen": 267714560 }, { "epoch": 0.08, "learning_rate": 0.0007425155466399198, "loss": 1.6038, "theoretical_loss": 4.231049236800907, "tokens_seen": 267780096 }, { "epoch": 0.08, "learning_rate": 0.0007424994984954864, "loss": 1.5656, "theoretical_loss": 4.230926340607109, "tokens_seen": 267845632 }, { "epoch": 0.08, "learning_rate": 0.0007424834503510532, "loss": 1.6124, "theoretical_loss": 4.230803482896918, "tokens_seen": 267911168 }, { "epoch": 0.08, "learning_rate": 0.0007424674022066198, "loss": 1.6741, "theoretical_loss": 4.230680663648872, "tokens_seen": 267976704 }, { "epoch": 0.08, "learning_rate": 0.0007424513540621866, "loss": 1.6197, "theoretical_loss": 4.230557882841531, "tokens_seen": 268042240 }, { "epoch": 0.08, "learning_rate": 0.0007424353059177532, "loss": 1.7328, "theoretical_loss": 4.2304351404534675, "tokens_seen": 268107776 }, { "epoch": 0.08, "learning_rate": 0.00074241925777332, "loss": 1.6919, "theoretical_loss": 4.230312436463272, "tokens_seen": 268173312 }, { "epoch": 0.08, "learning_rate": 0.0007424032096288866, "loss": 1.6116, "theoretical_loss": 4.230189770849556, "tokens_seen": 268238848 }, { "epoch": 0.08, "learning_rate": 0.0007423871614844534, "loss": 1.6098, "theoretical_loss": 4.230067143590943, "tokens_seen": 268304384 }, { "epoch": 0.08, "learning_rate": 0.00074237111334002, "loss": 1.5206, "theoretical_loss": 4.229944554666076, "tokens_seen": 268369920 }, { "epoch": 0.08, "learning_rate": 0.0007423550651955868, "loss": 1.6026, "theoretical_loss": 4.229822004053617, "tokens_seen": 268435456 }, { "epoch": 0.08, "learning_rate": 0.0007423390170511535, "loss": 1.6134, "theoretical_loss": 4.229699491732242, "tokens_seen": 268500992 }, { "epoch": 0.08, "learning_rate": 0.0007423229689067202, "loss": 1.5862, "theoretical_loss": 4.2295770176806435, "tokens_seen": 268566528 }, { "epoch": 0.08, "learning_rate": 0.0007423069207622869, "loss": 1.6335, "theoretical_loss": 4.229454581877535, "tokens_seen": 268632064 }, { "epoch": 0.08, "objective/train/docs_used": 95678, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.629549503326416, "objective/train/theoretical_loss": 4.229332184301644, "objective/train/tokens_used": 289157600, "theoretical_loss": 4.229332184301644, "tokens_seen": 268697600 }, { "epoch": 0.08, "learning_rate": 0.0007422908726178536, "loss": 1.6398, "theoretical_loss": 4.229332184301644, "tokens_seen": 268697600 }, { "epoch": 0.08, "learning_rate": 0.0007422748244734203, "loss": 1.4887, "theoretical_loss": 4.229209824931714, "tokens_seen": 268763136 }, { "epoch": 0.08, "learning_rate": 0.000742258776328987, "loss": 1.6316, "theoretical_loss": 4.22908750374651, "tokens_seen": 268828672 }, { "epoch": 0.08, "learning_rate": 0.0007422427281845537, "loss": 1.5414, "theoretical_loss": 4.22896522072481, "tokens_seen": 268894208 }, { "epoch": 0.08, "learning_rate": 0.0007422266800401204, "loss": 1.6666, "theoretical_loss": 4.22884297584541, "tokens_seen": 268959744 }, { "epoch": 0.08, "learning_rate": 0.0007422106318956871, "loss": 1.6226, "theoretical_loss": 4.228720769087124, "tokens_seen": 269025280 }, { "epoch": 0.08, "learning_rate": 0.0007421945837512538, "loss": 1.4824, "theoretical_loss": 4.22859860042878, "tokens_seen": 269090816 }, { "epoch": 0.08, "learning_rate": 0.0007421785356068205, "loss": 1.574, "theoretical_loss": 4.2284764698492285, "tokens_seen": 269156352 }, { "epoch": 0.08, "learning_rate": 0.0007421624874623872, "loss": 1.6074, "theoretical_loss": 4.22835437732733, "tokens_seen": 269221888 }, { "epoch": 0.08, "learning_rate": 0.0007421464393179539, "loss": 1.6837, "theoretical_loss": 4.228232322841967, "tokens_seen": 269287424 }, { "epoch": 0.08, "learning_rate": 0.0007421303911735206, "loss": 1.5557, "theoretical_loss": 4.228110306372036, "tokens_seen": 269352960 }, { "epoch": 0.08, "learning_rate": 0.0007421143430290873, "loss": 1.5514, "theoretical_loss": 4.227988327896453, "tokens_seen": 269418496 }, { "epoch": 0.08, "learning_rate": 0.000742098294884654, "loss": 1.6616, "theoretical_loss": 4.2278663873941476, "tokens_seen": 269484032 }, { "epoch": 0.08, "learning_rate": 0.0007420822467402207, "loss": 1.5448, "theoretical_loss": 4.227744484844068, "tokens_seen": 269549568 }, { "epoch": 0.08, "learning_rate": 0.0007420661985957874, "loss": 1.5115, "theoretical_loss": 4.227622620225182, "tokens_seen": 269615104 }, { "epoch": 0.08, "learning_rate": 0.0007420501504513541, "loss": 1.6058, "theoretical_loss": 4.227500793516466, "tokens_seen": 269680640 }, { "epoch": 0.08, "learning_rate": 0.0007420341023069208, "loss": 1.6111, "theoretical_loss": 4.227379004696923, "tokens_seen": 269746176 }, { "epoch": 0.08, "learning_rate": 0.0007420180541624875, "loss": 1.5459, "theoretical_loss": 4.227257253745566, "tokens_seen": 269811712 }, { "epoch": 0.08, "learning_rate": 0.0007420020060180542, "loss": 1.6436, "theoretical_loss": 4.227135540641426, "tokens_seen": 269877248 }, { "epoch": 0.08, "learning_rate": 0.0007419859578736209, "loss": 1.6299, "theoretical_loss": 4.227013865363553, "tokens_seen": 269942784 }, { "epoch": 0.08, "learning_rate": 0.0007419699097291876, "loss": 1.5194, "theoretical_loss": 4.226892227891012, "tokens_seen": 270008320 }, { "epoch": 0.08, "learning_rate": 0.0007419538615847543, "loss": 1.5954, "theoretical_loss": 4.2267706282028845, "tokens_seen": 270073856 }, { "epoch": 0.08, "learning_rate": 0.000741937813440321, "loss": 1.596, "theoretical_loss": 4.226649066278269, "tokens_seen": 270139392 }, { "epoch": 0.08, "learning_rate": 0.0007419217652958877, "loss": 1.7046, "theoretical_loss": 4.226527542096282, "tokens_seen": 270204928 }, { "epoch": 0.08, "learning_rate": 0.0007419057171514544, "loss": 1.6551, "theoretical_loss": 4.226406055636053, "tokens_seen": 270270464 }, { "epoch": 0.08, "learning_rate": 0.0007418896690070211, "loss": 1.6659, "theoretical_loss": 4.226284606876731, "tokens_seen": 270336000 }, { "epoch": 0.08, "learning_rate": 0.0007418736208625878, "loss": 1.5405, "theoretical_loss": 4.226163195797481, "tokens_seen": 270401536 }, { "epoch": 0.08, "learning_rate": 0.0007418575727181545, "loss": 1.6297, "theoretical_loss": 4.226041822377486, "tokens_seen": 270467072 }, { "epoch": 0.08, "learning_rate": 0.0007418415245737212, "loss": 1.5975, "theoretical_loss": 4.225920486595941, "tokens_seen": 270532608 }, { "epoch": 0.08, "learning_rate": 0.0007418254764292879, "loss": 1.6092, "theoretical_loss": 4.225799188432063, "tokens_seen": 270598144 }, { "epoch": 0.08, "learning_rate": 0.0007418094282848546, "loss": 1.665, "theoretical_loss": 4.225677927865083, "tokens_seen": 270663680 }, { "epoch": 0.08, "learning_rate": 0.0007417933801404213, "loss": 1.5838, "theoretical_loss": 4.225556704874247, "tokens_seen": 270729216 }, { "epoch": 0.08, "learning_rate": 0.000741777331995988, "loss": 1.6257, "theoretical_loss": 4.22543551943882, "tokens_seen": 270794752 }, { "epoch": 0.08, "learning_rate": 0.0007417612838515547, "loss": 1.5903, "theoretical_loss": 4.225314371538083, "tokens_seen": 270860288 }, { "epoch": 0.08, "learning_rate": 0.0007417452357071214, "loss": 1.5667, "theoretical_loss": 4.225193261151332, "tokens_seen": 270925824 }, { "epoch": 0.08, "learning_rate": 0.0007417291875626881, "loss": 1.5673, "theoretical_loss": 4.22507218825788, "tokens_seen": 270991360 }, { "epoch": 0.08, "learning_rate": 0.0007417131394182548, "loss": 1.5828, "theoretical_loss": 4.224951152837058, "tokens_seen": 271056896 }, { "epoch": 0.08, "learning_rate": 0.0007416970912738215, "loss": 1.6085, "theoretical_loss": 4.2248301548682115, "tokens_seen": 271122432 }, { "epoch": 0.08, "learning_rate": 0.0007416810431293882, "loss": 1.5875, "theoretical_loss": 4.2247091943307025, "tokens_seen": 271187968 }, { "epoch": 0.08, "learning_rate": 0.000741664994984955, "loss": 1.5357, "theoretical_loss": 4.224588271203911, "tokens_seen": 271253504 }, { "epoch": 0.08, "learning_rate": 0.0007416489468405216, "loss": 1.7111, "theoretical_loss": 4.22446738546723, "tokens_seen": 271319040 }, { "epoch": 0.08, "learning_rate": 0.0007416328986960883, "loss": 1.6451, "theoretical_loss": 4.224346537100074, "tokens_seen": 271384576 }, { "epoch": 0.08, "learning_rate": 0.000741616850551655, "loss": 1.4726, "theoretical_loss": 4.22422572608187, "tokens_seen": 271450112 }, { "epoch": 0.08, "learning_rate": 0.0007416008024072218, "loss": 1.5835, "theoretical_loss": 4.224104952392061, "tokens_seen": 271515648 }, { "epoch": 0.08, "learning_rate": 0.0007415847542627884, "loss": 1.6244, "theoretical_loss": 4.223984216010107, "tokens_seen": 271581184 }, { "epoch": 0.08, "learning_rate": 0.0007415687061183552, "loss": 1.6205, "theoretical_loss": 4.223863516915486, "tokens_seen": 271646720 }, { "epoch": 0.08, "learning_rate": 0.0007415526579739218, "loss": 1.5239, "theoretical_loss": 4.223742855087691, "tokens_seen": 271712256 }, { "epoch": 0.08, "learning_rate": 0.0007415366098294886, "loss": 1.5946, "theoretical_loss": 4.223622230506231, "tokens_seen": 271777792 }, { "epoch": 0.08, "learning_rate": 0.0007415205616850552, "loss": 1.7028, "theoretical_loss": 4.22350164315063, "tokens_seen": 271843328 }, { "epoch": 0.08, "learning_rate": 0.000741504513540622, "loss": 1.5819, "theoretical_loss": 4.22338109300043, "tokens_seen": 271908864 }, { "epoch": 0.08, "objective/train/docs_used": 96867, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5667804479599, "objective/train/theoretical_loss": 4.22326058003519, "objective/train/tokens_used": 292434400, "theoretical_loss": 4.22326058003519, "tokens_seen": 271974400 }, { "epoch": 0.08, "learning_rate": 0.0007414884653961886, "loss": 1.5744, "theoretical_loss": 4.22326058003519, "tokens_seen": 271974400 }, { "epoch": 0.08, "learning_rate": 0.0007414724172517554, "loss": 1.6947, "theoretical_loss": 4.223140104234482, "tokens_seen": 272039936 }, { "epoch": 0.08, "learning_rate": 0.000741456369107322, "loss": 1.5442, "theoretical_loss": 4.2230196655778975, "tokens_seen": 272105472 }, { "epoch": 0.08, "learning_rate": 0.0007414403209628888, "loss": 1.707, "theoretical_loss": 4.222899264045042, "tokens_seen": 272171008 }, { "epoch": 0.08, "learning_rate": 0.0007414242728184554, "loss": 1.635, "theoretical_loss": 4.2227788996155375, "tokens_seen": 272236544 }, { "epoch": 0.08, "learning_rate": 0.0007414082246740222, "loss": 1.6033, "theoretical_loss": 4.222658572269022, "tokens_seen": 272302080 }, { "epoch": 0.08, "learning_rate": 0.0007413921765295888, "loss": 1.6143, "theoretical_loss": 4.222538281985151, "tokens_seen": 272367616 }, { "epoch": 0.08, "learning_rate": 0.0007413761283851556, "loss": 1.6715, "theoretical_loss": 4.2224180287435935, "tokens_seen": 272433152 }, { "epoch": 0.08, "learning_rate": 0.0007413600802407222, "loss": 1.5306, "theoretical_loss": 4.222297812524037, "tokens_seen": 272498688 }, { "epoch": 0.08, "learning_rate": 0.000741344032096289, "loss": 1.6365, "theoretical_loss": 4.222177633306183, "tokens_seen": 272564224 }, { "epoch": 0.08, "learning_rate": 0.0007413279839518556, "loss": 1.657, "theoretical_loss": 4.222057491069751, "tokens_seen": 272629760 }, { "epoch": 0.08, "learning_rate": 0.0007413119358074224, "loss": 1.5467, "theoretical_loss": 4.221937385794474, "tokens_seen": 272695296 }, { "epoch": 0.08, "learning_rate": 0.000741295887662989, "loss": 1.5622, "theoretical_loss": 4.221817317460104, "tokens_seen": 272760832 }, { "epoch": 0.08, "learning_rate": 0.0007412798395185557, "loss": 1.53, "theoretical_loss": 4.221697286046407, "tokens_seen": 272826368 }, { "epoch": 0.08, "learning_rate": 0.0007412637913741224, "loss": 1.4924, "theoretical_loss": 4.221577291533165, "tokens_seen": 272891904 }, { "epoch": 0.08, "learning_rate": 0.0007412477432296891, "loss": 1.583, "theoretical_loss": 4.221457333900176, "tokens_seen": 272957440 }, { "epoch": 0.08, "learning_rate": 0.0007412316950852558, "loss": 1.5246, "theoretical_loss": 4.221337413127254, "tokens_seen": 273022976 }, { "epoch": 0.08, "learning_rate": 0.0007412156469408225, "loss": 1.42, "theoretical_loss": 4.221217529194231, "tokens_seen": 273088512 }, { "epoch": 0.08, "learning_rate": 0.0007411995987963892, "loss": 1.6641, "theoretical_loss": 4.22109768208095, "tokens_seen": 273154048 }, { "epoch": 0.08, "learning_rate": 0.0007411835506519559, "loss": 1.5788, "theoretical_loss": 4.220977871767275, "tokens_seen": 273219584 }, { "epoch": 0.08, "learning_rate": 0.0007411675025075226, "loss": 1.6302, "theoretical_loss": 4.220858098233082, "tokens_seen": 273285120 }, { "epoch": 0.08, "learning_rate": 0.0007411514543630893, "loss": 1.5654, "theoretical_loss": 4.220738361458266, "tokens_seen": 273350656 }, { "epoch": 0.08, "learning_rate": 0.000741135406218656, "loss": 1.5599, "theoretical_loss": 4.220618661422735, "tokens_seen": 273416192 }, { "epoch": 0.08, "learning_rate": 0.0007411193580742227, "loss": 1.6897, "theoretical_loss": 4.2204989981064145, "tokens_seen": 273481728 }, { "epoch": 0.08, "learning_rate": 0.0007411033099297894, "loss": 1.5683, "theoretical_loss": 4.220379371489246, "tokens_seen": 273547264 }, { "epoch": 0.08, "learning_rate": 0.0007410872617853561, "loss": 1.5457, "theoretical_loss": 4.220259781551185, "tokens_seen": 273612800 }, { "epoch": 0.08, "learning_rate": 0.0007410712136409228, "loss": 1.6194, "theoretical_loss": 4.2201402282722045, "tokens_seen": 273678336 }, { "epoch": 0.08, "learning_rate": 0.0007410551654964895, "loss": 1.6366, "theoretical_loss": 4.2200207116322925, "tokens_seen": 273743872 }, { "epoch": 0.08, "learning_rate": 0.0007410391173520562, "loss": 1.6136, "theoretical_loss": 4.219901231611452, "tokens_seen": 273809408 }, { "epoch": 0.08, "learning_rate": 0.0007410230692076229, "loss": 1.6047, "theoretical_loss": 4.219781788189703, "tokens_seen": 273874944 }, { "epoch": 0.08, "learning_rate": 0.0007410070210631896, "loss": 1.4999, "theoretical_loss": 4.2196623813470815, "tokens_seen": 273940480 }, { "epoch": 0.08, "learning_rate": 0.0007409909729187563, "loss": 1.6752, "theoretical_loss": 4.219543011063637, "tokens_seen": 274006016 }, { "epoch": 0.08, "learning_rate": 0.000740974924774323, "loss": 1.7212, "theoretical_loss": 4.219423677319437, "tokens_seen": 274071552 }, { "epoch": 0.08, "learning_rate": 0.0007409588766298897, "loss": 1.6442, "theoretical_loss": 4.219304380094562, "tokens_seen": 274137088 }, { "epoch": 0.08, "learning_rate": 0.0007409428284854564, "loss": 1.6143, "theoretical_loss": 4.2191851193691114, "tokens_seen": 274202624 }, { "epoch": 0.08, "learning_rate": 0.0007409267803410231, "loss": 1.6529, "theoretical_loss": 4.219065895123197, "tokens_seen": 274268160 }, { "epoch": 0.08, "learning_rate": 0.0007409107321965898, "loss": 1.6902, "theoretical_loss": 4.218946707336949, "tokens_seen": 274333696 }, { "epoch": 0.08, "learning_rate": 0.0007408946840521565, "loss": 1.5987, "theoretical_loss": 4.2188275559905115, "tokens_seen": 274399232 }, { "epoch": 0.08, "learning_rate": 0.0007408786359077232, "loss": 1.527, "theoretical_loss": 4.218708441064044, "tokens_seen": 274464768 }, { "epoch": 0.08, "learning_rate": 0.0007408625877632899, "loss": 1.5728, "theoretical_loss": 4.218589362537723, "tokens_seen": 274530304 }, { "epoch": 0.08, "learning_rate": 0.0007408465396188566, "loss": 1.5688, "theoretical_loss": 4.218470320391739, "tokens_seen": 274595840 }, { "epoch": 0.08, "learning_rate": 0.0007408304914744233, "loss": 1.5903, "theoretical_loss": 4.218351314606298, "tokens_seen": 274661376 }, { "epoch": 0.08, "learning_rate": 0.00074081444332999, "loss": 1.6266, "theoretical_loss": 4.218232345161622, "tokens_seen": 274726912 }, { "epoch": 0.08, "learning_rate": 0.0007407983951855567, "loss": 1.5244, "theoretical_loss": 4.21811341203795, "tokens_seen": 274792448 }, { "epoch": 0.08, "learning_rate": 0.0007407823470411234, "loss": 1.6264, "theoretical_loss": 4.217994515215534, "tokens_seen": 274857984 }, { "epoch": 0.08, "learning_rate": 0.0007407662988966901, "loss": 1.6983, "theoretical_loss": 4.217875654674642, "tokens_seen": 274923520 }, { "epoch": 0.08, "learning_rate": 0.0007407502507522568, "loss": 1.5818, "theoretical_loss": 4.217756830395559, "tokens_seen": 274989056 }, { "epoch": 0.08, "learning_rate": 0.0007407342026078235, "loss": 1.6596, "theoretical_loss": 4.217638042358583, "tokens_seen": 275054592 }, { "epoch": 0.08, "learning_rate": 0.0007407181544633902, "loss": 1.6276, "theoretical_loss": 4.21751929054403, "tokens_seen": 275120128 }, { "epoch": 0.08, "learning_rate": 0.0007407021063189569, "loss": 1.6848, "theoretical_loss": 4.2174005749322285, "tokens_seen": 275185664 }, { "epoch": 0.08, "objective/train/docs_used": 97725, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.526691198348999, "objective/train/theoretical_loss": 4.2172818955035245, "objective/train/tokens_used": 295711200, "theoretical_loss": 4.2172818955035245, "tokens_seen": 275251200 }, { "epoch": 0.08, "learning_rate": 0.0007406860581745236, "loss": 1.5652, "theoretical_loss": 4.2172818955035245, "tokens_seen": 275251200 }, { "epoch": 0.08, "learning_rate": 0.0007406700100300903, "loss": 1.5369, "theoretical_loss": 4.217163252238279, "tokens_seen": 275316736 }, { "epoch": 0.08, "learning_rate": 0.000740653961885657, "loss": 1.4076, "theoretical_loss": 4.217044645116869, "tokens_seen": 275382272 }, { "epoch": 0.08, "learning_rate": 0.0007406379137412237, "loss": 1.6474, "theoretical_loss": 4.216926074119684, "tokens_seen": 275447808 }, { "epoch": 0.08, "learning_rate": 0.0007406218655967904, "loss": 1.5036, "theoretical_loss": 4.216807539227132, "tokens_seen": 275513344 }, { "epoch": 0.08, "learning_rate": 0.0007406058174523571, "loss": 1.6137, "theoretical_loss": 4.2166890404196335, "tokens_seen": 275578880 }, { "epoch": 0.08, "learning_rate": 0.0007405897693079238, "loss": 1.6188, "theoretical_loss": 4.216570577677627, "tokens_seen": 275644416 }, { "epoch": 0.08, "learning_rate": 0.0007405737211634905, "loss": 1.6735, "theoretical_loss": 4.216452150981566, "tokens_seen": 275709952 }, { "epoch": 0.08, "learning_rate": 0.0007405576730190572, "loss": 1.4854, "theoretical_loss": 4.216333760311916, "tokens_seen": 275775488 }, { "epoch": 0.08, "learning_rate": 0.0007405416248746239, "loss": 1.6522, "theoretical_loss": 4.216215405649161, "tokens_seen": 275841024 }, { "epoch": 0.08, "learning_rate": 0.0007405255767301906, "loss": 1.5212, "theoretical_loss": 4.2160970869738, "tokens_seen": 275906560 }, { "epoch": 0.08, "learning_rate": 0.0007405095285857573, "loss": 1.7428, "theoretical_loss": 4.215978804266345, "tokens_seen": 275972096 }, { "epoch": 0.08, "learning_rate": 0.000740493480441324, "loss": 1.6548, "theoretical_loss": 4.215860557507324, "tokens_seen": 276037632 }, { "epoch": 0.08, "learning_rate": 0.0007404774322968907, "loss": 1.4961, "theoretical_loss": 4.215742346677283, "tokens_seen": 276103168 }, { "epoch": 0.08, "learning_rate": 0.0007404613841524574, "loss": 1.5168, "theoretical_loss": 4.215624171756779, "tokens_seen": 276168704 }, { "epoch": 0.08, "learning_rate": 0.0007404453360080241, "loss": 1.5582, "theoretical_loss": 4.215506032726386, "tokens_seen": 276234240 }, { "epoch": 0.08, "learning_rate": 0.0007404292878635908, "loss": 1.6843, "theoretical_loss": 4.215387929566694, "tokens_seen": 276299776 }, { "epoch": 0.08, "learning_rate": 0.0007404132397191575, "loss": 1.6185, "theoretical_loss": 4.215269862258307, "tokens_seen": 276365312 }, { "epoch": 0.08, "learning_rate": 0.0007403971915747242, "loss": 1.4809, "theoretical_loss": 4.215151830781843, "tokens_seen": 276430848 }, { "epoch": 0.08, "learning_rate": 0.0007403811434302909, "loss": 1.4883, "theoretical_loss": 4.2150338351179375, "tokens_seen": 276496384 }, { "epoch": 0.08, "learning_rate": 0.0007403650952858576, "loss": 1.6069, "theoretical_loss": 4.21491587524724, "tokens_seen": 276561920 }, { "epoch": 0.08, "learning_rate": 0.0007403490471414243, "loss": 1.7038, "theoretical_loss": 4.214797951150415, "tokens_seen": 276627456 }, { "epoch": 0.08, "learning_rate": 0.000740332998996991, "loss": 1.581, "theoretical_loss": 4.214680062808142, "tokens_seen": 276692992 }, { "epoch": 0.08, "learning_rate": 0.0007403169508525577, "loss": 1.542, "theoretical_loss": 4.214562210201114, "tokens_seen": 276758528 }, { "epoch": 0.08, "learning_rate": 0.0007403009027081244, "loss": 1.4822, "theoretical_loss": 4.214444393310042, "tokens_seen": 276824064 }, { "epoch": 0.08, "learning_rate": 0.0007402848545636911, "loss": 1.5861, "theoretical_loss": 4.2143266121156495, "tokens_seen": 276889600 }, { "epoch": 0.08, "learning_rate": 0.0007402688064192578, "loss": 1.6427, "theoretical_loss": 4.2142088665986766, "tokens_seen": 276955136 }, { "epoch": 0.08, "learning_rate": 0.0007402527582748245, "loss": 1.535, "theoretical_loss": 4.214091156739878, "tokens_seen": 277020672 }, { "epoch": 0.08, "learning_rate": 0.0007402367101303912, "loss": 1.6281, "theoretical_loss": 4.213973482520021, "tokens_seen": 277086208 }, { "epoch": 0.08, "learning_rate": 0.0007402206619859579, "loss": 1.6517, "theoretical_loss": 4.213855843919891, "tokens_seen": 277151744 }, { "epoch": 0.08, "learning_rate": 0.0007402046138415247, "loss": 1.6089, "theoretical_loss": 4.2137382409202875, "tokens_seen": 277217280 }, { "epoch": 0.08, "learning_rate": 0.0007401885656970913, "loss": 1.6273, "theoretical_loss": 4.2136206735020245, "tokens_seen": 277282816 }, { "epoch": 0.08, "learning_rate": 0.000740172517552658, "loss": 1.5309, "theoretical_loss": 4.21350314164593, "tokens_seen": 277348352 }, { "epoch": 0.08, "learning_rate": 0.0007401564694082247, "loss": 1.612, "theoretical_loss": 4.213385645332848, "tokens_seen": 277413888 }, { "epoch": 0.08, "learning_rate": 0.0007401404212637915, "loss": 1.7019, "theoretical_loss": 4.213268184543638, "tokens_seen": 277479424 }, { "epoch": 0.08, "learning_rate": 0.0007401243731193581, "loss": 1.5525, "theoretical_loss": 4.213150759259172, "tokens_seen": 277544960 }, { "epoch": 0.08, "learning_rate": 0.0007401083249749249, "loss": 1.601, "theoretical_loss": 4.213033369460339, "tokens_seen": 277610496 }, { "epoch": 0.08, "learning_rate": 0.0007400922768304915, "loss": 1.55, "theoretical_loss": 4.212916015128041, "tokens_seen": 277676032 }, { "epoch": 0.08, "learning_rate": 0.0007400762286860583, "loss": 1.6188, "theoretical_loss": 4.212798696243197, "tokens_seen": 277741568 }, { "epoch": 0.08, "learning_rate": 0.0007400601805416249, "loss": 1.6865, "theoretical_loss": 4.212681412786738, "tokens_seen": 277807104 }, { "epoch": 0.08, "learning_rate": 0.0007400441323971917, "loss": 1.6119, "theoretical_loss": 4.212564164739613, "tokens_seen": 277872640 }, { "epoch": 0.08, "learning_rate": 0.0007400280842527583, "loss": 1.653, "theoretical_loss": 4.212446952082783, "tokens_seen": 277938176 }, { "epoch": 0.08, "learning_rate": 0.0007400120361083251, "loss": 1.5911, "theoretical_loss": 4.212329774797225, "tokens_seen": 278003712 }, { "epoch": 0.08, "learning_rate": 0.0007399959879638917, "loss": 1.5933, "theoretical_loss": 4.212212632863931, "tokens_seen": 278069248 }, { "epoch": 0.08, "learning_rate": 0.0007399799398194585, "loss": 1.5824, "theoretical_loss": 4.212095526263905, "tokens_seen": 278134784 }, { "epoch": 0.08, "learning_rate": 0.0007399638916750251, "loss": 1.593, "theoretical_loss": 4.211978454978171, "tokens_seen": 278200320 }, { "epoch": 0.08, "learning_rate": 0.0007399478435305919, "loss": 1.4985, "theoretical_loss": 4.211861418987762, "tokens_seen": 278265856 }, { "epoch": 0.08, "learning_rate": 0.0007399317953861585, "loss": 1.7031, "theoretical_loss": 4.211744418273728, "tokens_seen": 278331392 }, { "epoch": 0.08, "learning_rate": 0.0007399157472417253, "loss": 1.4938, "theoretical_loss": 4.211627452817136, "tokens_seen": 278396928 }, { "epoch": 0.08, "learning_rate": 0.0007398996990972919, "loss": 1.5638, "theoretical_loss": 4.211510522599063, "tokens_seen": 278462464 }, { "epoch": 0.08, "objective/train/docs_used": 98910, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5239521265029907, "objective/train/theoretical_loss": 4.2113936276006045, "objective/train/tokens_used": 298988000, "theoretical_loss": 4.2113936276006045, "tokens_seen": 278528000 }, { "epoch": 0.08, "learning_rate": 0.0007398836509528586, "loss": 1.5001, "theoretical_loss": 4.2113936276006045, "tokens_seen": 278528000 }, { "epoch": 0.08, "learning_rate": 0.0007398676028084253, "loss": 1.4508, "theoretical_loss": 4.211276767802868, "tokens_seen": 278593536 }, { "epoch": 0.08, "learning_rate": 0.000739851554663992, "loss": 1.5051, "theoretical_loss": 4.211159943186978, "tokens_seen": 278659072 }, { "epoch": 0.08, "learning_rate": 0.0007398355065195587, "loss": 1.5914, "theoretical_loss": 4.211043153734071, "tokens_seen": 278724608 }, { "epoch": 0.08, "learning_rate": 0.0007398194583751254, "loss": 1.5629, "theoretical_loss": 4.2109263994253, "tokens_seen": 278790144 }, { "epoch": 0.08, "learning_rate": 0.0007398034102306921, "loss": 1.5097, "theoretical_loss": 4.21080968024183, "tokens_seen": 278855680 }, { "epoch": 0.08, "learning_rate": 0.0007397873620862588, "loss": 1.6827, "theoretical_loss": 4.210692996164845, "tokens_seen": 278921216 }, { "epoch": 0.08, "learning_rate": 0.0007397713139418255, "loss": 1.539, "theoretical_loss": 4.210576347175539, "tokens_seen": 278986752 }, { "epoch": 0.08, "learning_rate": 0.0007397552657973922, "loss": 1.6543, "theoretical_loss": 4.210459733255123, "tokens_seen": 279052288 }, { "epoch": 0.08, "learning_rate": 0.0007397392176529589, "loss": 1.5957, "theoretical_loss": 4.2103431543848195, "tokens_seen": 279117824 }, { "epoch": 0.08, "learning_rate": 0.0007397231695085256, "loss": 1.5742, "theoretical_loss": 4.210226610545872, "tokens_seen": 279183360 }, { "epoch": 0.08, "learning_rate": 0.0007397071213640923, "loss": 1.6426, "theoretical_loss": 4.21011010171953, "tokens_seen": 279248896 }, { "epoch": 0.08, "learning_rate": 0.000739691073219659, "loss": 1.5525, "theoretical_loss": 4.209993627887064, "tokens_seen": 279314432 }, { "epoch": 0.08, "learning_rate": 0.0007396750250752257, "loss": 1.5869, "theoretical_loss": 4.209877189029756, "tokens_seen": 279379968 }, { "epoch": 0.08, "learning_rate": 0.0007396589769307924, "loss": 1.6383, "theoretical_loss": 4.209760785128903, "tokens_seen": 279445504 }, { "epoch": 0.08, "learning_rate": 0.0007396429287863591, "loss": 1.638, "theoretical_loss": 4.2096444161658155, "tokens_seen": 279511040 }, { "epoch": 0.08, "learning_rate": 0.0007396268806419258, "loss": 1.5023, "theoretical_loss": 4.209528082121819, "tokens_seen": 279576576 }, { "epoch": 0.08, "learning_rate": 0.0007396108324974925, "loss": 1.5516, "theoretical_loss": 4.209411782978256, "tokens_seen": 279642112 }, { "epoch": 0.08, "learning_rate": 0.0007395947843530592, "loss": 1.5677, "theoretical_loss": 4.209295518716477, "tokens_seen": 279707648 }, { "epoch": 0.08, "learning_rate": 0.0007395787362086259, "loss": 1.5189, "theoretical_loss": 4.209179289317854, "tokens_seen": 279773184 }, { "epoch": 0.08, "learning_rate": 0.0007395626880641926, "loss": 1.5775, "theoretical_loss": 4.209063094763767, "tokens_seen": 279838720 }, { "epoch": 0.08, "learning_rate": 0.0007395466399197593, "loss": 1.5707, "theoretical_loss": 4.208946935035616, "tokens_seen": 279904256 }, { "epoch": 0.08, "learning_rate": 0.000739530591775326, "loss": 1.5592, "theoretical_loss": 4.2088308101148115, "tokens_seen": 279969792 }, { "epoch": 0.08, "learning_rate": 0.0007395145436308927, "loss": 1.6149, "theoretical_loss": 4.208714719982778, "tokens_seen": 280035328 }, { "epoch": 0.08, "learning_rate": 0.0007394984954864594, "loss": 1.5742, "theoretical_loss": 4.208598664620958, "tokens_seen": 280100864 }, { "epoch": 0.08, "learning_rate": 0.0007394824473420261, "loss": 1.6079, "theoretical_loss": 4.208482644010804, "tokens_seen": 280166400 }, { "epoch": 0.08, "learning_rate": 0.0007394663991975928, "loss": 1.5717, "theoretical_loss": 4.2083666581337855, "tokens_seen": 280231936 }, { "epoch": 0.08, "learning_rate": 0.0007394503510531595, "loss": 1.5169, "theoretical_loss": 4.208250706971385, "tokens_seen": 280297472 }, { "epoch": 0.08, "learning_rate": 0.0007394343029087262, "loss": 1.6327, "theoretical_loss": 4.208134790505099, "tokens_seen": 280363008 }, { "epoch": 0.08, "learning_rate": 0.0007394182547642929, "loss": 1.6121, "theoretical_loss": 4.20801890871644, "tokens_seen": 280428544 }, { "epoch": 0.08, "learning_rate": 0.0007394022066198596, "loss": 1.5979, "theoretical_loss": 4.207903061586931, "tokens_seen": 280494080 }, { "epoch": 0.09, "learning_rate": 0.0007393861584754263, "loss": 1.6794, "theoretical_loss": 4.207787249098113, "tokens_seen": 280559616 }, { "epoch": 0.09, "learning_rate": 0.000739370110330993, "loss": 1.5258, "theoretical_loss": 4.20767147123154, "tokens_seen": 280625152 }, { "epoch": 0.09, "learning_rate": 0.0007393540621865597, "loss": 1.6724, "theoretical_loss": 4.20755572796878, "tokens_seen": 280690688 }, { "epoch": 0.09, "learning_rate": 0.0007393380140421264, "loss": 1.5395, "theoretical_loss": 4.207440019291413, "tokens_seen": 280756224 }, { "epoch": 0.09, "learning_rate": 0.0007393219658976931, "loss": 1.6244, "theoretical_loss": 4.2073243451810365, "tokens_seen": 280821760 }, { "epoch": 0.09, "learning_rate": 0.0007393059177532598, "loss": 1.6882, "theoretical_loss": 4.20720870561926, "tokens_seen": 280887296 }, { "epoch": 0.09, "learning_rate": 0.0007392898696088265, "loss": 1.4866, "theoretical_loss": 4.2070931005877075, "tokens_seen": 280952832 }, { "epoch": 0.09, "learning_rate": 0.0007392738214643932, "loss": 1.5088, "theoretical_loss": 4.206977530068017, "tokens_seen": 281018368 }, { "epoch": 0.09, "learning_rate": 0.0007392577733199599, "loss": 1.583, "theoretical_loss": 4.206861994041842, "tokens_seen": 281083904 }, { "epoch": 0.09, "learning_rate": 0.0007392417251755266, "loss": 1.5661, "theoretical_loss": 4.206746492490847, "tokens_seen": 281149440 }, { "epoch": 0.09, "learning_rate": 0.0007392256770310933, "loss": 1.6379, "theoretical_loss": 4.206631025396714, "tokens_seen": 281214976 }, { "epoch": 0.09, "learning_rate": 0.00073920962888666, "loss": 1.587, "theoretical_loss": 4.206515592741135, "tokens_seen": 281280512 }, { "epoch": 0.09, "learning_rate": 0.0007391935807422267, "loss": 1.6539, "theoretical_loss": 4.20640019450582, "tokens_seen": 281346048 }, { "epoch": 0.09, "learning_rate": 0.0007391775325977934, "loss": 1.6555, "theoretical_loss": 4.206284830672491, "tokens_seen": 281411584 }, { "epoch": 0.09, "learning_rate": 0.0007391614844533601, "loss": 1.6364, "theoretical_loss": 4.206169501222885, "tokens_seen": 281477120 }, { "epoch": 0.09, "learning_rate": 0.0007391454363089268, "loss": 1.5291, "theoretical_loss": 4.20605420613875, "tokens_seen": 281542656 }, { "epoch": 0.09, "learning_rate": 0.0007391293881644935, "loss": 1.5918, "theoretical_loss": 4.205938945401851, "tokens_seen": 281608192 }, { "epoch": 0.09, "learning_rate": 0.0007391133400200602, "loss": 1.5979, "theoretical_loss": 4.205823718993966, "tokens_seen": 281673728 }, { "epoch": 0.09, "learning_rate": 0.0007390972918756269, "loss": 1.6177, "theoretical_loss": 4.2057085268968875, "tokens_seen": 281739264 }, { "epoch": 0.09, "objective/train/docs_used": 99932, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.517197608947754, "objective/train/theoretical_loss": 4.205593369092421, "objective/train/tokens_used": 302264800, "theoretical_loss": 4.205593369092421, "tokens_seen": 281804800 }, { "epoch": 0.09, "learning_rate": 0.0007390812437311936, "loss": 1.5729, "theoretical_loss": 4.205593369092421, "tokens_seen": 281804800 }, { "epoch": 0.09, "learning_rate": 0.0007390651955867603, "loss": 1.6341, "theoretical_loss": 4.205478245562386, "tokens_seen": 281870336 }, { "epoch": 0.09, "learning_rate": 0.000739049147442327, "loss": 1.6788, "theoretical_loss": 4.205363156288615, "tokens_seen": 281935872 }, { "epoch": 0.09, "learning_rate": 0.0007390330992978937, "loss": 1.621, "theoretical_loss": 4.205248101252957, "tokens_seen": 282001408 }, { "epoch": 0.09, "learning_rate": 0.0007390170511534604, "loss": 1.544, "theoretical_loss": 4.205133080437273, "tokens_seen": 282066944 }, { "epoch": 0.09, "learning_rate": 0.0007390010030090271, "loss": 1.6125, "theoretical_loss": 4.205018093823437, "tokens_seen": 282132480 }, { "epoch": 0.09, "learning_rate": 0.0007389849548645938, "loss": 1.5787, "theoretical_loss": 4.204903141393338, "tokens_seen": 282198016 }, { "epoch": 0.09, "learning_rate": 0.0007389689067201605, "loss": 1.6345, "theoretical_loss": 4.204788223128879, "tokens_seen": 282263552 }, { "epoch": 0.09, "learning_rate": 0.0007389528585757272, "loss": 1.5378, "theoretical_loss": 4.204673339011976, "tokens_seen": 282329088 }, { "epoch": 0.09, "learning_rate": 0.000738936810431294, "loss": 1.6958, "theoretical_loss": 4.204558489024561, "tokens_seen": 282394624 }, { "epoch": 0.09, "learning_rate": 0.0007389207622868606, "loss": 1.5529, "theoretical_loss": 4.204443673148575, "tokens_seen": 282460160 }, { "epoch": 0.09, "learning_rate": 0.0007389047141424273, "loss": 1.4651, "theoretical_loss": 4.204328891365977, "tokens_seen": 282525696 }, { "epoch": 0.09, "learning_rate": 0.000738888665997994, "loss": 1.551, "theoretical_loss": 4.2042141436587395, "tokens_seen": 282591232 }, { "epoch": 0.09, "learning_rate": 0.0007388726178535607, "loss": 1.5604, "theoretical_loss": 4.204099430008846, "tokens_seen": 282656768 }, { "epoch": 0.09, "learning_rate": 0.0007388565697091274, "loss": 1.5569, "theoretical_loss": 4.203984750398296, "tokens_seen": 282722304 }, { "epoch": 0.09, "learning_rate": 0.0007388405215646942, "loss": 1.6475, "theoretical_loss": 4.203870104809101, "tokens_seen": 282787840 }, { "epoch": 0.09, "learning_rate": 0.0007388244734202608, "loss": 1.5684, "theoretical_loss": 4.203755493223289, "tokens_seen": 282853376 }, { "epoch": 0.09, "learning_rate": 0.0007388084252758276, "loss": 1.7048, "theoretical_loss": 4.203640915622898, "tokens_seen": 282918912 }, { "epoch": 0.09, "learning_rate": 0.0007387923771313942, "loss": 1.6332, "theoretical_loss": 4.203526371989983, "tokens_seen": 282984448 }, { "epoch": 0.09, "learning_rate": 0.000738776328986961, "loss": 1.6873, "theoretical_loss": 4.203411862306609, "tokens_seen": 283049984 }, { "epoch": 0.09, "learning_rate": 0.0007387602808425276, "loss": 1.6564, "theoretical_loss": 4.203297386554859, "tokens_seen": 283115520 }, { "epoch": 0.09, "learning_rate": 0.0007387442326980944, "loss": 1.6088, "theoretical_loss": 4.203182944716826, "tokens_seen": 283181056 }, { "epoch": 0.09, "learning_rate": 0.0007387281845536611, "loss": 1.6262, "theoretical_loss": 4.203068536774619, "tokens_seen": 283246592 }, { "epoch": 0.09, "learning_rate": 0.0007387121364092278, "loss": 1.5406, "theoretical_loss": 4.202954162710357, "tokens_seen": 283312128 }, { "epoch": 0.09, "learning_rate": 0.0007386960882647945, "loss": 1.731, "theoretical_loss": 4.202839822506178, "tokens_seen": 283377664 }, { "epoch": 0.09, "learning_rate": 0.0007386800401203612, "loss": 1.6566, "theoretical_loss": 4.202725516144228, "tokens_seen": 283443200 }, { "epoch": 0.09, "learning_rate": 0.0007386639919759279, "loss": 1.6085, "theoretical_loss": 4.202611243606671, "tokens_seen": 283508736 }, { "epoch": 0.09, "learning_rate": 0.0007386479438314946, "loss": 1.7107, "theoretical_loss": 4.202497004875681, "tokens_seen": 283574272 }, { "epoch": 0.09, "learning_rate": 0.0007386318956870613, "loss": 1.6096, "theoretical_loss": 4.202382799933449, "tokens_seen": 283639808 }, { "epoch": 0.09, "learning_rate": 0.000738615847542628, "loss": 1.4388, "theoretical_loss": 4.2022686287621775, "tokens_seen": 283705344 }, { "epoch": 0.09, "learning_rate": 0.0007385997993981946, "loss": 1.588, "theoretical_loss": 4.20215449134408, "tokens_seen": 283770880 }, { "epoch": 0.09, "learning_rate": 0.0007385837512537614, "loss": 1.634, "theoretical_loss": 4.202040387661389, "tokens_seen": 283836416 }, { "epoch": 0.09, "learning_rate": 0.000738567703109328, "loss": 1.6546, "theoretical_loss": 4.201926317696347, "tokens_seen": 283901952 }, { "epoch": 0.09, "learning_rate": 0.0007385516549648948, "loss": 1.656, "theoretical_loss": 4.201812281431209, "tokens_seen": 283967488 }, { "epoch": 0.09, "learning_rate": 0.0007385356068204614, "loss": 1.6949, "theoretical_loss": 4.201698278848246, "tokens_seen": 284033024 }, { "epoch": 0.09, "learning_rate": 0.0007385195586760282, "loss": 1.5907, "theoretical_loss": 4.201584309929743, "tokens_seen": 284098560 }, { "epoch": 0.09, "learning_rate": 0.0007385035105315948, "loss": 1.6153, "theoretical_loss": 4.201470374657993, "tokens_seen": 284164096 }, { "epoch": 0.09, "learning_rate": 0.0007384874623871615, "loss": 1.6676, "theoretical_loss": 4.201356473015308, "tokens_seen": 284229632 }, { "epoch": 0.09, "learning_rate": 0.0007384714142427282, "loss": 1.6801, "theoretical_loss": 4.201242604984013, "tokens_seen": 284295168 }, { "epoch": 0.09, "learning_rate": 0.0007384553660982949, "loss": 1.7536, "theoretical_loss": 4.201128770546442, "tokens_seen": 284360704 }, { "epoch": 0.09, "learning_rate": 0.0007384393179538616, "loss": 1.6957, "theoretical_loss": 4.201014969684948, "tokens_seen": 284426240 }, { "epoch": 0.09, "learning_rate": 0.0007384232698094283, "loss": 1.6393, "theoretical_loss": 4.200901202381893, "tokens_seen": 284491776 }, { "epoch": 0.09, "learning_rate": 0.000738407221664995, "loss": 1.6103, "theoretical_loss": 4.200787468619653, "tokens_seen": 284557312 }, { "epoch": 0.09, "learning_rate": 0.0007383911735205617, "loss": 1.6311, "theoretical_loss": 4.200673768380621, "tokens_seen": 284622848 }, { "epoch": 0.09, "learning_rate": 0.0007383751253761284, "loss": 1.6207, "theoretical_loss": 4.200560101647198, "tokens_seen": 284688384 }, { "epoch": 0.09, "learning_rate": 0.0007383590772316951, "loss": 1.5322, "theoretical_loss": 4.2004464684018, "tokens_seen": 284753920 }, { "epoch": 0.09, "learning_rate": 0.0007383430290872618, "loss": 1.5648, "theoretical_loss": 4.200332868626861, "tokens_seen": 284819456 }, { "epoch": 0.09, "learning_rate": 0.0007383269809428285, "loss": 1.4811, "theoretical_loss": 4.200219302304821, "tokens_seen": 284884992 }, { "epoch": 0.09, "learning_rate": 0.0007383109327983952, "loss": 1.5884, "theoretical_loss": 4.200105769418138, "tokens_seen": 284950528 }, { "epoch": 0.09, "learning_rate": 0.0007382948846539619, "loss": 1.6323, "theoretical_loss": 4.199992269949281, "tokens_seen": 285016064 }, { "epoch": 0.09, "objective/train/docs_used": 101193, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5234239101409912, "objective/train/theoretical_loss": 4.199878803880733, "objective/train/tokens_used": 305541600, "theoretical_loss": 4.199878803880733, "tokens_seen": 285081600 }, { "epoch": 0.09, "learning_rate": 0.0007382788365095286, "loss": 1.4773, "theoretical_loss": 4.199878803880733, "tokens_seen": 285081600 }, { "epoch": 0.09, "learning_rate": 0.0007382627883650953, "loss": 1.4435, "theoretical_loss": 4.1997653711949905, "tokens_seen": 285147136 }, { "epoch": 0.09, "learning_rate": 0.000738246740220662, "loss": 1.5471, "theoretical_loss": 4.199651971874562, "tokens_seen": 285212672 }, { "epoch": 0.09, "learning_rate": 0.0007382306920762287, "loss": 1.5007, "theoretical_loss": 4.199538605901973, "tokens_seen": 285278208 }, { "epoch": 0.09, "learning_rate": 0.0007382146439317954, "loss": 1.6037, "theoretical_loss": 4.199425273259756, "tokens_seen": 285343744 }, { "epoch": 0.09, "learning_rate": 0.0007381985957873621, "loss": 1.5669, "theoretical_loss": 4.199311973930461, "tokens_seen": 285409280 }, { "epoch": 0.09, "learning_rate": 0.0007381825476429288, "loss": 1.5726, "theoretical_loss": 4.199198707896651, "tokens_seen": 285474816 }, { "epoch": 0.09, "learning_rate": 0.0007381664994984955, "loss": 1.5804, "theoretical_loss": 4.199085475140899, "tokens_seen": 285540352 }, { "epoch": 0.09, "learning_rate": 0.0007381504513540622, "loss": 1.6743, "theoretical_loss": 4.1989722756457954, "tokens_seen": 285605888 }, { "epoch": 0.09, "learning_rate": 0.0007381344032096289, "loss": 1.5639, "theoretical_loss": 4.198859109393941, "tokens_seen": 285671424 }, { "epoch": 0.09, "learning_rate": 0.0007381183550651956, "loss": 1.5105, "theoretical_loss": 4.198745976367949, "tokens_seen": 285736960 }, { "epoch": 0.09, "learning_rate": 0.0007381023069207623, "loss": 1.5629, "theoretical_loss": 4.198632876550449, "tokens_seen": 285802496 }, { "epoch": 0.09, "learning_rate": 0.000738086258776329, "loss": 1.5286, "theoretical_loss": 4.19851980992408, "tokens_seen": 285868032 }, { "epoch": 0.09, "learning_rate": 0.0007380702106318957, "loss": 1.6526, "theoretical_loss": 4.198406776471497, "tokens_seen": 285933568 }, { "epoch": 0.09, "learning_rate": 0.0007380541624874624, "loss": 1.7248, "theoretical_loss": 4.198293776175365, "tokens_seen": 285999104 }, { "epoch": 0.09, "learning_rate": 0.0007380381143430291, "loss": 1.4311, "theoretical_loss": 4.198180809018366, "tokens_seen": 286064640 }, { "epoch": 0.09, "learning_rate": 0.0007380220661985958, "loss": 1.5445, "theoretical_loss": 4.198067874983192, "tokens_seen": 286130176 }, { "epoch": 0.09, "learning_rate": 0.0007380060180541625, "loss": 1.58, "theoretical_loss": 4.197954974052546, "tokens_seen": 286195712 }, { "epoch": 0.09, "learning_rate": 0.0007379899699097292, "loss": 1.5693, "theoretical_loss": 4.19784210620915, "tokens_seen": 286261248 }, { "epoch": 0.09, "learning_rate": 0.0007379739217652959, "loss": 1.6604, "theoretical_loss": 4.197729271435736, "tokens_seen": 286326784 }, { "epoch": 0.09, "learning_rate": 0.0007379578736208626, "loss": 1.6563, "theoretical_loss": 4.197616469715046, "tokens_seen": 286392320 }, { "epoch": 0.09, "learning_rate": 0.0007379418254764293, "loss": 1.75, "theoretical_loss": 4.19750370102984, "tokens_seen": 286457856 }, { "epoch": 0.09, "learning_rate": 0.000737925777331996, "loss": 1.7021, "theoretical_loss": 4.197390965362887, "tokens_seen": 286523392 }, { "epoch": 0.09, "learning_rate": 0.0007379097291875627, "loss": 1.567, "theoretical_loss": 4.197278262696971, "tokens_seen": 286588928 }, { "epoch": 0.09, "learning_rate": 0.0007378936810431294, "loss": 1.5716, "theoretical_loss": 4.197165593014889, "tokens_seen": 286654464 }, { "epoch": 0.09, "learning_rate": 0.0007378776328986961, "loss": 1.6708, "theoretical_loss": 4.197052956299449, "tokens_seen": 286720000 }, { "epoch": 0.09, "learning_rate": 0.0007378615847542628, "loss": 1.5052, "theoretical_loss": 4.196940352533474, "tokens_seen": 286785536 }, { "epoch": 0.09, "learning_rate": 0.0007378455366098295, "loss": 1.5812, "theoretical_loss": 4.196827781699799, "tokens_seen": 286851072 }, { "epoch": 0.09, "learning_rate": 0.0007378294884653962, "loss": 1.6132, "theoretical_loss": 4.196715243781273, "tokens_seen": 286916608 }, { "epoch": 0.09, "learning_rate": 0.0007378134403209629, "loss": 1.6742, "theoretical_loss": 4.196602738760754, "tokens_seen": 286982144 }, { "epoch": 0.09, "learning_rate": 0.0007377973921765296, "loss": 1.6104, "theoretical_loss": 4.196490266621119, "tokens_seen": 287047680 }, { "epoch": 0.09, "learning_rate": 0.0007377813440320963, "loss": 1.6694, "theoretical_loss": 4.196377827345252, "tokens_seen": 287113216 }, { "epoch": 0.09, "learning_rate": 0.000737765295887663, "loss": 1.5387, "theoretical_loss": 4.196265420916053, "tokens_seen": 287178752 }, { "epoch": 0.09, "learning_rate": 0.0007377492477432297, "loss": 1.6054, "theoretical_loss": 4.196153047316433, "tokens_seen": 287244288 }, { "epoch": 0.09, "learning_rate": 0.0007377331995987964, "loss": 1.5317, "theoretical_loss": 4.196040706529319, "tokens_seen": 287309824 }, { "epoch": 0.09, "learning_rate": 0.0007377171514543631, "loss": 1.6415, "theoretical_loss": 4.195928398537647, "tokens_seen": 287375360 }, { "epoch": 0.09, "learning_rate": 0.0007377011033099298, "loss": 1.6341, "theoretical_loss": 4.195816123324367, "tokens_seen": 287440896 }, { "epoch": 0.09, "learning_rate": 0.0007376850551654965, "loss": 1.6622, "theoretical_loss": 4.195703880872443, "tokens_seen": 287506432 }, { "epoch": 0.09, "learning_rate": 0.0007376690070210632, "loss": 1.5524, "theoretical_loss": 4.195591671164852, "tokens_seen": 287571968 }, { "epoch": 0.09, "learning_rate": 0.0007376529588766299, "loss": 1.5814, "theoretical_loss": 4.195479494184581, "tokens_seen": 287637504 }, { "epoch": 0.09, "learning_rate": 0.0007376369107321966, "loss": 1.5762, "theoretical_loss": 4.195367349914631, "tokens_seen": 287703040 }, { "epoch": 0.09, "learning_rate": 0.0007376208625877633, "loss": 1.5798, "theoretical_loss": 4.195255238338016, "tokens_seen": 287768576 }, { "epoch": 0.09, "learning_rate": 0.00073760481444333, "loss": 1.5229, "theoretical_loss": 4.195143159437764, "tokens_seen": 287834112 }, { "epoch": 0.09, "learning_rate": 0.0007375887662988967, "loss": 1.5433, "theoretical_loss": 4.195031113196913, "tokens_seen": 287899648 }, { "epoch": 0.09, "learning_rate": 0.0007375727181544634, "loss": 1.6659, "theoretical_loss": 4.194919099598516, "tokens_seen": 287965184 }, { "epoch": 0.09, "learning_rate": 0.0007375566700100301, "loss": 1.5775, "theoretical_loss": 4.194807118625636, "tokens_seen": 288030720 }, { "epoch": 0.09, "learning_rate": 0.0007375406218655968, "loss": 1.6525, "theoretical_loss": 4.194695170261353, "tokens_seen": 288096256 }, { "epoch": 0.09, "learning_rate": 0.0007375245737211636, "loss": 1.5722, "theoretical_loss": 4.194583254488754, "tokens_seen": 288161792 }, { "epoch": 0.09, "learning_rate": 0.0007375085255767302, "loss": 1.5539, "theoretical_loss": 4.194471371290944, "tokens_seen": 288227328 }, { "epoch": 0.09, "learning_rate": 0.000737492477432297, "loss": 1.6119, "theoretical_loss": 4.194359520651036, "tokens_seen": 288292864 }, { "epoch": 0.09, "objective/train/docs_used": 102045, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5714504718780518, "objective/train/theoretical_loss": 4.1942477025521585, "objective/train/tokens_used": 308818400, "theoretical_loss": 4.1942477025521585, "tokens_seen": 288358400 }, { "epoch": 0.09, "learning_rate": 0.0007374764292878637, "loss": 1.5811, "theoretical_loss": 4.1942477025521585, "tokens_seen": 288358400 }, { "epoch": 0.09, "learning_rate": 0.0007374603811434304, "loss": 1.6291, "theoretical_loss": 4.194135916977452, "tokens_seen": 288423936 }, { "epoch": 0.09, "learning_rate": 0.000737444332998997, "loss": 1.6362, "theoretical_loss": 4.19402416391007, "tokens_seen": 288489472 }, { "epoch": 0.09, "learning_rate": 0.0007374282848545638, "loss": 1.5195, "theoretical_loss": 4.193912443333177, "tokens_seen": 288555008 }, { "epoch": 0.09, "learning_rate": 0.0007374122367101305, "loss": 1.474, "theoretical_loss": 4.193800755229951, "tokens_seen": 288620544 }, { "epoch": 0.09, "learning_rate": 0.0007373961885656972, "loss": 1.6996, "theoretical_loss": 4.193689099583582, "tokens_seen": 288686080 }, { "epoch": 0.09, "learning_rate": 0.0007373801404212639, "loss": 1.5586, "theoretical_loss": 4.193577476377275, "tokens_seen": 288751616 }, { "epoch": 0.09, "learning_rate": 0.0007373640922768306, "loss": 1.5385, "theoretical_loss": 4.193465885594242, "tokens_seen": 288817152 }, { "epoch": 0.09, "learning_rate": 0.0007373480441323973, "loss": 1.5508, "theoretical_loss": 4.1933543272177145, "tokens_seen": 288882688 }, { "epoch": 0.09, "learning_rate": 0.000737331995987964, "loss": 1.6673, "theoretical_loss": 4.193242801230931, "tokens_seen": 288948224 }, { "epoch": 0.09, "learning_rate": 0.0007373159478435307, "loss": 1.6181, "theoretical_loss": 4.193131307617145, "tokens_seen": 289013760 }, { "epoch": 0.09, "learning_rate": 0.0007372998996990974, "loss": 1.6803, "theoretical_loss": 4.193019846359622, "tokens_seen": 289079296 }, { "epoch": 0.09, "learning_rate": 0.0007372838515546641, "loss": 1.6104, "theoretical_loss": 4.192908417441639, "tokens_seen": 289144832 }, { "epoch": 0.09, "learning_rate": 0.0007372678034102308, "loss": 1.6341, "theoretical_loss": 4.192797020846487, "tokens_seen": 289210368 }, { "epoch": 0.09, "learning_rate": 0.0007372517552657975, "loss": 1.5669, "theoretical_loss": 4.192685656557468, "tokens_seen": 289275904 }, { "epoch": 0.09, "learning_rate": 0.0007372357071213642, "loss": 1.6607, "theoretical_loss": 4.1925743245578975, "tokens_seen": 289341440 }, { "epoch": 0.09, "learning_rate": 0.0007372196589769309, "loss": 1.5765, "theoretical_loss": 4.1924630248311034, "tokens_seen": 289406976 }, { "epoch": 0.09, "learning_rate": 0.0007372036108324975, "loss": 1.48, "theoretical_loss": 4.192351757360425, "tokens_seen": 289472512 }, { "epoch": 0.09, "learning_rate": 0.0007371875626880643, "loss": 1.5329, "theoretical_loss": 4.1922405221292145, "tokens_seen": 289538048 }, { "epoch": 0.09, "learning_rate": 0.0007371715145436309, "loss": 1.4819, "theoretical_loss": 4.192129319120838, "tokens_seen": 289603584 }, { "epoch": 0.09, "learning_rate": 0.0007371554663991977, "loss": 1.6302, "theoretical_loss": 4.1920181483186685, "tokens_seen": 289669120 }, { "epoch": 0.09, "learning_rate": 0.0007371394182547643, "loss": 1.5838, "theoretical_loss": 4.1919070097060995, "tokens_seen": 289734656 }, { "epoch": 0.09, "learning_rate": 0.0007371233701103311, "loss": 1.5787, "theoretical_loss": 4.19179590326653, "tokens_seen": 289800192 }, { "epoch": 0.09, "learning_rate": 0.0007371073219658977, "loss": 1.5751, "theoretical_loss": 4.191684828983375, "tokens_seen": 289865728 }, { "epoch": 0.09, "learning_rate": 0.0007370912738214645, "loss": 1.664, "theoretical_loss": 4.191573786840061, "tokens_seen": 289931264 }, { "epoch": 0.09, "learning_rate": 0.0007370752256770311, "loss": 1.5749, "theoretical_loss": 4.191462776820025, "tokens_seen": 289996800 }, { "epoch": 0.09, "learning_rate": 0.0007370591775325978, "loss": 1.604, "theoretical_loss": 4.191351798906719, "tokens_seen": 290062336 }, { "epoch": 0.09, "learning_rate": 0.0007370431293881645, "loss": 1.7533, "theoretical_loss": 4.191240853083604, "tokens_seen": 290127872 }, { "epoch": 0.09, "learning_rate": 0.0007370270812437312, "loss": 1.6221, "theoretical_loss": 4.191129939334159, "tokens_seen": 290193408 }, { "epoch": 0.09, "learning_rate": 0.0007370110330992979, "loss": 1.5681, "theoretical_loss": 4.191019057641868, "tokens_seen": 290258944 }, { "epoch": 0.09, "learning_rate": 0.0007369949849548646, "loss": 1.6289, "theoretical_loss": 4.1909082079902324, "tokens_seen": 290324480 }, { "epoch": 0.09, "learning_rate": 0.0007369789368104313, "loss": 1.7017, "theoretical_loss": 4.190797390362764, "tokens_seen": 290390016 }, { "epoch": 0.09, "learning_rate": 0.000736962888665998, "loss": 1.5707, "theoretical_loss": 4.190686604742986, "tokens_seen": 290455552 }, { "epoch": 0.09, "learning_rate": 0.0007369468405215647, "loss": 1.6355, "theoretical_loss": 4.190575851114437, "tokens_seen": 290521088 }, { "epoch": 0.09, "learning_rate": 0.0007369307923771314, "loss": 1.7072, "theoretical_loss": 4.190465129460662, "tokens_seen": 290586624 }, { "epoch": 0.09, "learning_rate": 0.0007369147442326981, "loss": 1.6748, "theoretical_loss": 4.190354439765224, "tokens_seen": 290652160 }, { "epoch": 0.09, "learning_rate": 0.0007368986960882648, "loss": 1.5803, "theoretical_loss": 4.190243782011696, "tokens_seen": 290717696 }, { "epoch": 0.09, "learning_rate": 0.0007368826479438315, "loss": 1.6931, "theoretical_loss": 4.190133156183663, "tokens_seen": 290783232 }, { "epoch": 0.09, "learning_rate": 0.0007368665997993982, "loss": 1.5436, "theoretical_loss": 4.190022562264721, "tokens_seen": 290848768 }, { "epoch": 0.09, "learning_rate": 0.0007368505516549649, "loss": 1.6706, "theoretical_loss": 4.189912000238479, "tokens_seen": 290914304 }, { "epoch": 0.09, "learning_rate": 0.0007368345035105316, "loss": 1.5295, "theoretical_loss": 4.189801470088559, "tokens_seen": 290979840 }, { "epoch": 0.09, "learning_rate": 0.0007368184553660983, "loss": 1.6353, "theoretical_loss": 4.189690971798596, "tokens_seen": 291045376 }, { "epoch": 0.09, "learning_rate": 0.000736802407221665, "loss": 1.6281, "theoretical_loss": 4.189580505352232, "tokens_seen": 291110912 }, { "epoch": 0.09, "learning_rate": 0.0007367863590772317, "loss": 1.6095, "theoretical_loss": 4.189470070733129, "tokens_seen": 291176448 }, { "epoch": 0.09, "learning_rate": 0.0007367703109327984, "loss": 1.6304, "theoretical_loss": 4.189359667924952, "tokens_seen": 291241984 }, { "epoch": 0.09, "learning_rate": 0.0007367542627883651, "loss": 1.6004, "theoretical_loss": 4.189249296911386, "tokens_seen": 291307520 }, { "epoch": 0.09, "learning_rate": 0.0007367382146439318, "loss": 1.6946, "theoretical_loss": 4.189138957676125, "tokens_seen": 291373056 }, { "epoch": 0.09, "learning_rate": 0.0007367221664994985, "loss": 1.7297, "theoretical_loss": 4.1890286502028715, "tokens_seen": 291438592 }, { "epoch": 0.09, "learning_rate": 0.0007367061183550652, "loss": 1.6308, "theoretical_loss": 4.188918374475347, "tokens_seen": 291504128 }, { "epoch": 0.09, "learning_rate": 0.0007366900702106319, "loss": 1.6745, "theoretical_loss": 4.188808130477279, "tokens_seen": 291569664 }, { "epoch": 0.09, "objective/train/docs_used": 103221, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6348395347595215, "objective/train/theoretical_loss": 4.1886979181924096, "objective/train/tokens_used": 312095200, "theoretical_loss": 4.1886979181924096, "tokens_seen": 291635200 }, { "epoch": 0.09, "learning_rate": 0.0007366740220661986, "loss": 1.647, "theoretical_loss": 4.1886979181924096, "tokens_seen": 291635200 }, { "epoch": 0.09, "learning_rate": 0.0007366579739217653, "loss": 1.7113, "theoretical_loss": 4.188587737604493, "tokens_seen": 291700736 }, { "epoch": 0.09, "learning_rate": 0.000736641925777332, "loss": 1.6427, "theoretical_loss": 4.188477588697295, "tokens_seen": 291766272 }, { "epoch": 0.09, "learning_rate": 0.0007366258776328987, "loss": 1.5969, "theoretical_loss": 4.188367471454593, "tokens_seen": 291831808 }, { "epoch": 0.09, "learning_rate": 0.0007366098294884654, "loss": 1.4803, "theoretical_loss": 4.188257385860177, "tokens_seen": 291897344 }, { "epoch": 0.09, "learning_rate": 0.0007365937813440321, "loss": 1.6042, "theoretical_loss": 4.188147331897849, "tokens_seen": 291962880 }, { "epoch": 0.09, "learning_rate": 0.0007365777331995988, "loss": 1.6471, "theoretical_loss": 4.1880373095514205, "tokens_seen": 292028416 }, { "epoch": 0.09, "learning_rate": 0.0007365616850551655, "loss": 1.6034, "theoretical_loss": 4.18792731880472, "tokens_seen": 292093952 }, { "epoch": 0.09, "learning_rate": 0.0007365456369107322, "loss": 1.5614, "theoretical_loss": 4.187817359641581, "tokens_seen": 292159488 }, { "epoch": 0.09, "learning_rate": 0.0007365295887662989, "loss": 1.5428, "theoretical_loss": 4.1877074320458565, "tokens_seen": 292225024 }, { "epoch": 0.09, "learning_rate": 0.0007365135406218656, "loss": 1.5486, "theoretical_loss": 4.187597536001406, "tokens_seen": 292290560 }, { "epoch": 0.09, "learning_rate": 0.0007364974924774323, "loss": 1.5498, "theoretical_loss": 4.187487671492102, "tokens_seen": 292356096 }, { "epoch": 0.09, "learning_rate": 0.000736481444332999, "loss": 1.6196, "theoretical_loss": 4.18737783850183, "tokens_seen": 292421632 }, { "epoch": 0.09, "learning_rate": 0.0007364653961885657, "loss": 1.6716, "theoretical_loss": 4.187268037014486, "tokens_seen": 292487168 }, { "epoch": 0.09, "learning_rate": 0.0007364493480441324, "loss": 1.6067, "theoretical_loss": 4.187158267013979, "tokens_seen": 292552704 }, { "epoch": 0.09, "learning_rate": 0.0007364332998996991, "loss": 1.517, "theoretical_loss": 4.18704852848423, "tokens_seen": 292618240 }, { "epoch": 0.09, "learning_rate": 0.0007364172517552658, "loss": 1.6312, "theoretical_loss": 4.18693882140917, "tokens_seen": 292683776 }, { "epoch": 0.09, "learning_rate": 0.0007364012036108325, "loss": 1.4959, "theoretical_loss": 4.186829145772743, "tokens_seen": 292749312 }, { "epoch": 0.09, "learning_rate": 0.0007363851554663992, "loss": 1.5707, "theoretical_loss": 4.186719501558905, "tokens_seen": 292814848 }, { "epoch": 0.09, "learning_rate": 0.0007363691073219659, "loss": 1.5797, "theoretical_loss": 4.186609888751623, "tokens_seen": 292880384 }, { "epoch": 0.09, "learning_rate": 0.0007363530591775326, "loss": 1.6421, "theoretical_loss": 4.186500307334878, "tokens_seen": 292945920 }, { "epoch": 0.09, "learning_rate": 0.0007363370110330993, "loss": 1.6258, "theoretical_loss": 4.186390757292659, "tokens_seen": 293011456 }, { "epoch": 0.09, "learning_rate": 0.0007363209628886661, "loss": 1.5723, "theoretical_loss": 4.18628123860897, "tokens_seen": 293076992 }, { "epoch": 0.09, "learning_rate": 0.0007363049147442327, "loss": 1.638, "theoretical_loss": 4.186171751267825, "tokens_seen": 293142528 }, { "epoch": 0.09, "learning_rate": 0.0007362888665997995, "loss": 1.6085, "theoretical_loss": 4.186062295253249, "tokens_seen": 293208064 }, { "epoch": 0.09, "learning_rate": 0.0007362728184553661, "loss": 1.6619, "theoretical_loss": 4.185952870549283, "tokens_seen": 293273600 }, { "epoch": 0.09, "learning_rate": 0.0007362567703109329, "loss": 1.5903, "theoretical_loss": 4.185843477139974, "tokens_seen": 293339136 }, { "epoch": 0.09, "learning_rate": 0.0007362407221664995, "loss": 1.555, "theoretical_loss": 4.185734115009383, "tokens_seen": 293404672 }, { "epoch": 0.09, "learning_rate": 0.0007362246740220663, "loss": 1.5025, "theoretical_loss": 4.185624784141585, "tokens_seen": 293470208 }, { "epoch": 0.09, "learning_rate": 0.000736208625877633, "loss": 1.5115, "theoretical_loss": 4.185515484520664, "tokens_seen": 293535744 }, { "epoch": 0.09, "learning_rate": 0.0007361925777331997, "loss": 1.6593, "theoretical_loss": 4.1854062161307155, "tokens_seen": 293601280 }, { "epoch": 0.09, "learning_rate": 0.0007361765295887663, "loss": 1.7188, "theoretical_loss": 4.185296978955848, "tokens_seen": 293666816 }, { "epoch": 0.09, "learning_rate": 0.0007361604814443331, "loss": 1.5893, "theoretical_loss": 4.185187772980181, "tokens_seen": 293732352 }, { "epoch": 0.09, "learning_rate": 0.0007361444332998997, "loss": 1.4278, "theoretical_loss": 4.185078598187846, "tokens_seen": 293797888 }, { "epoch": 0.09, "learning_rate": 0.0007361283851554665, "loss": 1.4631, "theoretical_loss": 4.184969454562985, "tokens_seen": 293863424 }, { "epoch": 0.09, "learning_rate": 0.0007361123370110332, "loss": 1.5732, "theoretical_loss": 4.1848603420897525, "tokens_seen": 293928960 }, { "epoch": 0.09, "learning_rate": 0.0007360962888665999, "loss": 1.5025, "theoretical_loss": 4.184751260752316, "tokens_seen": 293994496 }, { "epoch": 0.09, "learning_rate": 0.0007360802407221666, "loss": 1.5542, "theoretical_loss": 4.18464221053485, "tokens_seen": 294060032 }, { "epoch": 0.09, "learning_rate": 0.0007360641925777333, "loss": 1.6657, "theoretical_loss": 4.184533191421547, "tokens_seen": 294125568 }, { "epoch": 0.09, "learning_rate": 0.0007360481444333, "loss": 1.7267, "theoretical_loss": 4.184424203396606, "tokens_seen": 294191104 }, { "epoch": 0.09, "learning_rate": 0.0007360320962888667, "loss": 1.4338, "theoretical_loss": 4.184315246444239, "tokens_seen": 294256640 }, { "epoch": 0.09, "learning_rate": 0.0007360160481444334, "loss": 1.5626, "theoretical_loss": 4.18420632054867, "tokens_seen": 294322176 }, { "epoch": 0.09, "learning_rate": 0.0007360000000000001, "loss": 1.6043, "theoretical_loss": 4.1840974256941355, "tokens_seen": 294387712 }, { "epoch": 0.09, "learning_rate": 0.0007359839518555668, "loss": 1.5946, "theoretical_loss": 4.183988561864879, "tokens_seen": 294453248 }, { "epoch": 0.09, "learning_rate": 0.0007359679037111335, "loss": 1.5876, "theoretical_loss": 4.183879729045163, "tokens_seen": 294518784 }, { "epoch": 0.09, "learning_rate": 0.0007359518555667002, "loss": 1.6075, "theoretical_loss": 4.183770927219255, "tokens_seen": 294584320 }, { "epoch": 0.09, "learning_rate": 0.0007359358074222669, "loss": 1.5569, "theoretical_loss": 4.183662156371436, "tokens_seen": 294649856 }, { "epoch": 0.09, "learning_rate": 0.0007359197592778336, "loss": 1.6115, "theoretical_loss": 4.183553416485999, "tokens_seen": 294715392 }, { "epoch": 0.09, "learning_rate": 0.0007359037111334003, "loss": 1.5446, "theoretical_loss": 4.183444707547249, "tokens_seen": 294780928 }, { "epoch": 0.09, "learning_rate": 0.000735887662988967, "loss": 1.6602, "theoretical_loss": 4.1833360295395, "tokens_seen": 294846464 }, { "epoch": 0.09, "objective/train/docs_used": 104159, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.72190523147583, "objective/train/theoretical_loss": 4.183227382447081, "objective/train/tokens_used": 315372000, "theoretical_loss": 4.183227382447081, "tokens_seen": 294912000 }, { "epoch": 0.09, "learning_rate": 0.0007358716148445337, "loss": 1.6478, "theoretical_loss": 4.183227382447081, "tokens_seen": 294912000 }, { "epoch": 0.09, "learning_rate": 0.0007358555667001004, "loss": 1.6181, "theoretical_loss": 4.183118766254328, "tokens_seen": 294977536 }, { "epoch": 0.09, "learning_rate": 0.0007358395185556671, "loss": 1.6082, "theoretical_loss": 4.183010180945593, "tokens_seen": 295043072 }, { "epoch": 0.09, "learning_rate": 0.0007358234704112338, "loss": 1.5291, "theoretical_loss": 4.182901626505237, "tokens_seen": 295108608 }, { "epoch": 0.09, "learning_rate": 0.0007358074222668004, "loss": 1.6135, "theoretical_loss": 4.182793102917632, "tokens_seen": 295174144 }, { "epoch": 0.09, "learning_rate": 0.0007357913741223672, "loss": 1.5915, "theoretical_loss": 4.182684610167162, "tokens_seen": 295239680 }, { "epoch": 0.09, "learning_rate": 0.0007357753259779338, "loss": 1.6561, "theoretical_loss": 4.1825761482382235, "tokens_seen": 295305216 }, { "epoch": 0.09, "learning_rate": 0.0007357592778335006, "loss": 1.7016, "theoretical_loss": 4.182467717115221, "tokens_seen": 295370752 }, { "epoch": 0.09, "learning_rate": 0.0007357432296890672, "loss": 1.6305, "theoretical_loss": 4.182359316782576, "tokens_seen": 295436288 }, { "epoch": 0.09, "learning_rate": 0.000735727181544634, "loss": 1.5485, "theoretical_loss": 4.1822509472247145, "tokens_seen": 295501824 }, { "epoch": 0.09, "learning_rate": 0.0007357111334002006, "loss": 1.6077, "theoretical_loss": 4.18214260842608, "tokens_seen": 295567360 }, { "epoch": 0.09, "learning_rate": 0.0007356950852557674, "loss": 1.5885, "theoretical_loss": 4.182034300371122, "tokens_seen": 295632896 }, { "epoch": 0.09, "learning_rate": 0.000735679037111334, "loss": 1.564, "theoretical_loss": 4.181926023044306, "tokens_seen": 295698432 }, { "epoch": 0.09, "learning_rate": 0.0007356629889669007, "loss": 1.5082, "theoretical_loss": 4.181817776430106, "tokens_seen": 295763968 }, { "epoch": 0.09, "learning_rate": 0.0007356469408224674, "loss": 1.711, "theoretical_loss": 4.181709560513008, "tokens_seen": 295829504 }, { "epoch": 0.09, "learning_rate": 0.0007356308926780341, "loss": 1.616, "theoretical_loss": 4.181601375277509, "tokens_seen": 295895040 }, { "epoch": 0.09, "learning_rate": 0.0007356148445336008, "loss": 1.6577, "theoretical_loss": 4.181493220708117, "tokens_seen": 295960576 }, { "epoch": 0.09, "learning_rate": 0.0007355987963891675, "loss": 1.6458, "theoretical_loss": 4.181385096789353, "tokens_seen": 296026112 }, { "epoch": 0.09, "learning_rate": 0.0007355827482447342, "loss": 1.6822, "theoretical_loss": 4.181277003505747, "tokens_seen": 296091648 }, { "epoch": 0.09, "learning_rate": 0.0007355667001003009, "loss": 1.5702, "theoretical_loss": 4.181168940841843, "tokens_seen": 296157184 }, { "epoch": 0.09, "learning_rate": 0.0007355506519558676, "loss": 1.6557, "theoretical_loss": 4.181060908782191, "tokens_seen": 296222720 }, { "epoch": 0.09, "learning_rate": 0.0007355346038114343, "loss": 1.5546, "theoretical_loss": 4.18095290731136, "tokens_seen": 296288256 }, { "epoch": 0.09, "learning_rate": 0.000735518555667001, "loss": 1.53, "theoretical_loss": 4.180844936413922, "tokens_seen": 296353792 }, { "epoch": 0.09, "learning_rate": 0.0007355025075225677, "loss": 1.562, "theoretical_loss": 4.180736996074465, "tokens_seen": 296419328 }, { "epoch": 0.09, "learning_rate": 0.0007354864593781344, "loss": 1.7042, "theoretical_loss": 4.180629086277589, "tokens_seen": 296484864 }, { "epoch": 0.09, "learning_rate": 0.0007354704112337011, "loss": 1.5895, "theoretical_loss": 4.180521207007902, "tokens_seen": 296550400 }, { "epoch": 0.09, "learning_rate": 0.0007354543630892678, "loss": 1.6513, "theoretical_loss": 4.180413358250024, "tokens_seen": 296615936 }, { "epoch": 0.09, "learning_rate": 0.0007354383149448345, "loss": 1.6866, "theoretical_loss": 4.180305539988588, "tokens_seen": 296681472 }, { "epoch": 0.09, "learning_rate": 0.0007354222668004012, "loss": 1.5783, "theoretical_loss": 4.180197752208235, "tokens_seen": 296747008 }, { "epoch": 0.09, "learning_rate": 0.0007354062186559679, "loss": 1.6043, "theoretical_loss": 4.1800899948936205, "tokens_seen": 296812544 }, { "epoch": 0.09, "learning_rate": 0.0007353901705115346, "loss": 1.565, "theoretical_loss": 4.179982268029409, "tokens_seen": 296878080 }, { "epoch": 0.09, "learning_rate": 0.0007353741223671013, "loss": 1.5138, "theoretical_loss": 4.179874571600277, "tokens_seen": 296943616 }, { "epoch": 0.09, "learning_rate": 0.000735358074222668, "loss": 1.6457, "theoretical_loss": 4.1797669055909115, "tokens_seen": 297009152 }, { "epoch": 0.09, "learning_rate": 0.0007353420260782347, "loss": 1.6078, "theoretical_loss": 4.1796592699860105, "tokens_seen": 297074688 }, { "epoch": 0.09, "learning_rate": 0.0007353259779338014, "loss": 1.691, "theoretical_loss": 4.179551664770283, "tokens_seen": 297140224 }, { "epoch": 0.09, "learning_rate": 0.0007353099297893681, "loss": 1.5473, "theoretical_loss": 4.1794440899284515, "tokens_seen": 297205760 }, { "epoch": 0.09, "learning_rate": 0.0007352938816449348, "loss": 1.5952, "theoretical_loss": 4.179336545445245, "tokens_seen": 297271296 }, { "epoch": 0.09, "learning_rate": 0.0007352778335005015, "loss": 1.5846, "theoretical_loss": 4.1792290313054075, "tokens_seen": 297336832 }, { "epoch": 0.09, "learning_rate": 0.0007352617853560682, "loss": 1.5803, "theoretical_loss": 4.179121547493692, "tokens_seen": 297402368 }, { "epoch": 0.09, "learning_rate": 0.0007352457372116349, "loss": 1.6163, "theoretical_loss": 4.179014093994862, "tokens_seen": 297467904 }, { "epoch": 0.09, "learning_rate": 0.0007352296890672016, "loss": 1.59, "theoretical_loss": 4.178906670793695, "tokens_seen": 297533440 }, { "epoch": 0.09, "learning_rate": 0.0007352136409227683, "loss": 1.6357, "theoretical_loss": 4.178799277874977, "tokens_seen": 297598976 }, { "epoch": 0.09, "learning_rate": 0.000735197592778335, "loss": 1.6383, "theoretical_loss": 4.178691915223505, "tokens_seen": 297664512 }, { "epoch": 0.09, "learning_rate": 0.0007351815446339017, "loss": 1.5541, "theoretical_loss": 4.178584582824088, "tokens_seen": 297730048 }, { "epoch": 0.09, "learning_rate": 0.0007351654964894684, "loss": 1.7037, "theoretical_loss": 4.1784772806615464, "tokens_seen": 297795584 }, { "epoch": 0.09, "learning_rate": 0.0007351494483450351, "loss": 1.6525, "theoretical_loss": 4.178370008720709, "tokens_seen": 297861120 }, { "epoch": 0.09, "learning_rate": 0.0007351334002006018, "loss": 1.5659, "theoretical_loss": 4.178262766986418, "tokens_seen": 297926656 }, { "epoch": 0.09, "learning_rate": 0.0007351173520561686, "loss": 1.5114, "theoretical_loss": 4.178155555443525, "tokens_seen": 297992192 }, { "epoch": 0.09, "learning_rate": 0.0007351013039117352, "loss": 1.7272, "theoretical_loss": 4.178048374076894, "tokens_seen": 298057728 }, { "epoch": 0.09, "learning_rate": 0.000735085255767302, "loss": 1.6131, "theoretical_loss": 4.1779412228714, "tokens_seen": 298123264 }, { "epoch": 0.09, "objective/train/docs_used": 105423, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5215253829956055, "objective/train/theoretical_loss": 4.177834101811927, "objective/train/tokens_used": 318648800, "theoretical_loss": 4.177834101811927, "tokens_seen": 298188800 }, { "epoch": 0.09, "learning_rate": 0.0007350692076228686, "loss": 1.5549, "theoretical_loss": 4.177834101811927, "tokens_seen": 298188800 }, { "epoch": 0.09, "learning_rate": 0.0007350531594784354, "loss": 1.4696, "theoretical_loss": 4.1777270108833715, "tokens_seen": 298254336 }, { "epoch": 0.09, "learning_rate": 0.000735037111334002, "loss": 1.6553, "theoretical_loss": 4.177619950070639, "tokens_seen": 298319872 }, { "epoch": 0.09, "learning_rate": 0.0007350210631895688, "loss": 1.6148, "theoretical_loss": 4.177512919358649, "tokens_seen": 298385408 }, { "epoch": 0.09, "learning_rate": 0.0007350050150451354, "loss": 1.6731, "theoretical_loss": 4.17740591873233, "tokens_seen": 298450944 }, { "epoch": 0.09, "learning_rate": 0.0007349889669007022, "loss": 1.6382, "theoretical_loss": 4.177298948176619, "tokens_seen": 298516480 }, { "epoch": 0.09, "learning_rate": 0.0007349729187562688, "loss": 1.4472, "theoretical_loss": 4.17719200767647, "tokens_seen": 298582016 }, { "epoch": 0.09, "learning_rate": 0.0007349568706118356, "loss": 1.6397, "theoretical_loss": 4.177085097216842, "tokens_seen": 298647552 }, { "epoch": 0.09, "learning_rate": 0.0007349408224674022, "loss": 1.6669, "theoretical_loss": 4.1769782167827065, "tokens_seen": 298713088 }, { "epoch": 0.09, "learning_rate": 0.000734924774322969, "loss": 1.6121, "theoretical_loss": 4.176871366359047, "tokens_seen": 298778624 }, { "epoch": 0.09, "learning_rate": 0.0007349087261785356, "loss": 1.4917, "theoretical_loss": 4.176764545930858, "tokens_seen": 298844160 }, { "epoch": 0.09, "learning_rate": 0.0007348926780341024, "loss": 1.5772, "theoretical_loss": 4.176657755483143, "tokens_seen": 298909696 }, { "epoch": 0.09, "learning_rate": 0.000734876629889669, "loss": 1.6754, "theoretical_loss": 4.176550995000917, "tokens_seen": 298975232 }, { "epoch": 0.09, "learning_rate": 0.0007348605817452358, "loss": 1.6027, "theoretical_loss": 4.176444264469206, "tokens_seen": 299040768 }, { "epoch": 0.09, "learning_rate": 0.0007348445336008024, "loss": 1.7495, "theoretical_loss": 4.176337563873046, "tokens_seen": 299106304 }, { "epoch": 0.09, "learning_rate": 0.0007348284854563692, "loss": 1.6497, "theoretical_loss": 4.176230893197486, "tokens_seen": 299171840 }, { "epoch": 0.09, "learning_rate": 0.0007348124373119358, "loss": 1.51, "theoretical_loss": 4.1761242524275834, "tokens_seen": 299237376 }, { "epoch": 0.09, "learning_rate": 0.0007347963891675026, "loss": 1.5526, "theoretical_loss": 4.176017641548407, "tokens_seen": 299302912 }, { "epoch": 0.09, "learning_rate": 0.0007347803410230692, "loss": 1.7034, "theoretical_loss": 4.175911060545037, "tokens_seen": 299368448 }, { "epoch": 0.09, "learning_rate": 0.000734764292878636, "loss": 1.6063, "theoretical_loss": 4.175804509402562, "tokens_seen": 299433984 }, { "epoch": 0.09, "learning_rate": 0.0007347482447342027, "loss": 1.4916, "theoretical_loss": 4.175697988106085, "tokens_seen": 299499520 }, { "epoch": 0.09, "learning_rate": 0.0007347321965897694, "loss": 1.7321, "theoretical_loss": 4.175591496640718, "tokens_seen": 299565056 }, { "epoch": 0.09, "learning_rate": 0.000734716148445336, "loss": 1.5392, "theoretical_loss": 4.175485034991581, "tokens_seen": 299630592 }, { "epoch": 0.09, "learning_rate": 0.0007347001003009028, "loss": 1.5464, "theoretical_loss": 4.17537860314381, "tokens_seen": 299696128 }, { "epoch": 0.09, "learning_rate": 0.0007346840521564695, "loss": 1.5023, "theoretical_loss": 4.175272201082547, "tokens_seen": 299761664 }, { "epoch": 0.09, "learning_rate": 0.0007346680040120362, "loss": 1.5762, "theoretical_loss": 4.175165828792946, "tokens_seen": 299827200 }, { "epoch": 0.09, "learning_rate": 0.0007346519558676029, "loss": 1.6361, "theoretical_loss": 4.1750594862601735, "tokens_seen": 299892736 }, { "epoch": 0.09, "learning_rate": 0.0007346359077231696, "loss": 1.6891, "theoretical_loss": 4.1749531734694045, "tokens_seen": 299958272 }, { "epoch": 0.09, "learning_rate": 0.0007346198595787363, "loss": 1.5498, "theoretical_loss": 4.174846890405825, "tokens_seen": 300023808 }, { "epoch": 0.09, "learning_rate": 0.000734603811434303, "loss": 1.5768, "theoretical_loss": 4.174740637054632, "tokens_seen": 300089344 }, { "epoch": 0.09, "learning_rate": 0.0007345877632898697, "loss": 1.5761, "theoretical_loss": 4.174634413401034, "tokens_seen": 300154880 }, { "epoch": 0.09, "learning_rate": 0.0007345717151454364, "loss": 1.5583, "theoretical_loss": 4.174528219430247, "tokens_seen": 300220416 }, { "epoch": 0.09, "learning_rate": 0.0007345556670010031, "loss": 1.5313, "theoretical_loss": 4.174422055127502, "tokens_seen": 300285952 }, { "epoch": 0.09, "learning_rate": 0.0007345396188565698, "loss": 1.6252, "theoretical_loss": 4.174315920478037, "tokens_seen": 300351488 }, { "epoch": 0.09, "learning_rate": 0.0007345235707121365, "loss": 1.5832, "theoretical_loss": 4.174209815467102, "tokens_seen": 300417024 }, { "epoch": 0.09, "learning_rate": 0.0007345075225677032, "loss": 1.5806, "theoretical_loss": 4.174103740079958, "tokens_seen": 300482560 }, { "epoch": 0.09, "learning_rate": 0.0007344914744232699, "loss": 1.6372, "theoretical_loss": 4.1739976943018755, "tokens_seen": 300548096 }, { "epoch": 0.09, "learning_rate": 0.0007344754262788366, "loss": 1.7141, "theoretical_loss": 4.173891678118135, "tokens_seen": 300613632 }, { "epoch": 0.09, "learning_rate": 0.0007344593781344033, "loss": 1.592, "theoretical_loss": 4.173785691514031, "tokens_seen": 300679168 }, { "epoch": 0.09, "learning_rate": 0.00073444332998997, "loss": 1.6567, "theoretical_loss": 4.173679734474862, "tokens_seen": 300744704 }, { "epoch": 0.09, "learning_rate": 0.0007344272818455367, "loss": 1.5908, "theoretical_loss": 4.173573806985945, "tokens_seen": 300810240 }, { "epoch": 0.09, "learning_rate": 0.0007344112337011033, "loss": 1.6119, "theoretical_loss": 4.173467909032602, "tokens_seen": 300875776 }, { "epoch": 0.09, "learning_rate": 0.0007343951855566701, "loss": 1.6013, "theoretical_loss": 4.173362040600166, "tokens_seen": 300941312 }, { "epoch": 0.09, "learning_rate": 0.0007343791374122367, "loss": 1.6848, "theoretical_loss": 4.173256201673983, "tokens_seen": 301006848 }, { "epoch": 0.09, "learning_rate": 0.0007343630892678035, "loss": 1.5417, "theoretical_loss": 4.173150392239406, "tokens_seen": 301072384 }, { "epoch": 0.09, "learning_rate": 0.0007343470411233701, "loss": 1.5092, "theoretical_loss": 4.173044612281802, "tokens_seen": 301137920 }, { "epoch": 0.09, "learning_rate": 0.0007343309929789369, "loss": 1.5109, "theoretical_loss": 4.172938861786546, "tokens_seen": 301203456 }, { "epoch": 0.09, "learning_rate": 0.0007343149448345035, "loss": 1.5317, "theoretical_loss": 4.1728331407390264, "tokens_seen": 301268992 }, { "epoch": 0.09, "learning_rate": 0.0007342988966900703, "loss": 1.58, "theoretical_loss": 4.172727449124636, "tokens_seen": 301334528 }, { "epoch": 0.09, "learning_rate": 0.0007342828485456369, "loss": 1.625, "theoretical_loss": 4.172621786928786, "tokens_seen": 301400064 }, { "epoch": 0.09, "objective/train/docs_used": 106715, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5944786071777344, "objective/train/theoretical_loss": 4.17251615413689, "objective/train/tokens_used": 321925600, "theoretical_loss": 4.17251615413689, "tokens_seen": 301465600 }, { "epoch": 0.09, "learning_rate": 0.0007342668004012036, "loss": 1.6601, "theoretical_loss": 4.17251615413689, "tokens_seen": 301465600 }, { "epoch": 0.09, "learning_rate": 0.0007342507522567703, "loss": 1.5005, "theoretical_loss": 4.172410550734378, "tokens_seen": 301531136 }, { "epoch": 0.09, "learning_rate": 0.000734234704112337, "loss": 1.5192, "theoretical_loss": 4.172304976706689, "tokens_seen": 301596672 }, { "epoch": 0.09, "learning_rate": 0.0007342186559679037, "loss": 1.5185, "theoretical_loss": 4.17219943203927, "tokens_seen": 301662208 }, { "epoch": 0.09, "learning_rate": 0.0007342026078234704, "loss": 1.7129, "theoretical_loss": 4.172093916717581, "tokens_seen": 301727744 }, { "epoch": 0.09, "learning_rate": 0.0007341865596790371, "loss": 1.5986, "theoretical_loss": 4.171988430727091, "tokens_seen": 301793280 }, { "epoch": 0.09, "learning_rate": 0.0007341705115346038, "loss": 1.5291, "theoretical_loss": 4.17188297405328, "tokens_seen": 301858816 }, { "epoch": 0.09, "learning_rate": 0.0007341544633901705, "loss": 1.6157, "theoretical_loss": 4.171777546681638, "tokens_seen": 301924352 }, { "epoch": 0.09, "learning_rate": 0.0007341384152457372, "loss": 1.6012, "theoretical_loss": 4.171672148597665, "tokens_seen": 301989888 }, { "epoch": 0.09, "learning_rate": 0.0007341223671013039, "loss": 1.7156, "theoretical_loss": 4.171566779786872, "tokens_seen": 302055424 }, { "epoch": 0.09, "learning_rate": 0.0007341063189568706, "loss": 1.5974, "theoretical_loss": 4.17146144023478, "tokens_seen": 302120960 }, { "epoch": 0.09, "learning_rate": 0.0007340902708124373, "loss": 1.5717, "theoretical_loss": 4.171356129926921, "tokens_seen": 302186496 }, { "epoch": 0.09, "learning_rate": 0.000734074222668004, "loss": 1.647, "theoretical_loss": 4.171250848848835, "tokens_seen": 302252032 }, { "epoch": 0.09, "learning_rate": 0.0007340581745235707, "loss": 1.7013, "theoretical_loss": 4.171145596986076, "tokens_seen": 302317568 }, { "epoch": 0.09, "learning_rate": 0.0007340421263791374, "loss": 1.5124, "theoretical_loss": 4.171040374324204, "tokens_seen": 302383104 }, { "epoch": 0.09, "learning_rate": 0.0007340260782347041, "loss": 1.5927, "theoretical_loss": 4.170935180848793, "tokens_seen": 302448640 }, { "epoch": 0.09, "learning_rate": 0.0007340100300902708, "loss": 1.5633, "theoretical_loss": 4.170830016545425, "tokens_seen": 302514176 }, { "epoch": 0.09, "learning_rate": 0.0007339939819458375, "loss": 1.7269, "theoretical_loss": 4.1707248813996936, "tokens_seen": 302579712 }, { "epoch": 0.09, "learning_rate": 0.0007339779338014042, "loss": 1.6634, "theoretical_loss": 4.170619775397201, "tokens_seen": 302645248 }, { "epoch": 0.09, "learning_rate": 0.0007339618856569709, "loss": 1.7733, "theoretical_loss": 4.1705146985235615, "tokens_seen": 302710784 }, { "epoch": 0.09, "learning_rate": 0.0007339458375125376, "loss": 1.6171, "theoretical_loss": 4.170409650764397, "tokens_seen": 302776320 }, { "epoch": 0.09, "learning_rate": 0.0007339297893681043, "loss": 1.49, "theoretical_loss": 4.170304632105344, "tokens_seen": 302841856 }, { "epoch": 0.09, "learning_rate": 0.0007339137412236711, "loss": 1.6787, "theoretical_loss": 4.170199642532045, "tokens_seen": 302907392 }, { "epoch": 0.09, "learning_rate": 0.0007338976930792377, "loss": 1.5294, "theoretical_loss": 4.170094682030154, "tokens_seen": 302972928 }, { "epoch": 0.09, "learning_rate": 0.0007338816449348045, "loss": 1.5229, "theoretical_loss": 4.169989750585335, "tokens_seen": 303038464 }, { "epoch": 0.09, "learning_rate": 0.0007338655967903711, "loss": 1.6258, "theoretical_loss": 4.169884848183264, "tokens_seen": 303104000 }, { "epoch": 0.09, "learning_rate": 0.0007338495486459379, "loss": 1.5401, "theoretical_loss": 4.169779974809624, "tokens_seen": 303169536 }, { "epoch": 0.09, "learning_rate": 0.0007338335005015045, "loss": 1.6634, "theoretical_loss": 4.169675130450111, "tokens_seen": 303235072 }, { "epoch": 0.09, "learning_rate": 0.0007338174523570713, "loss": 1.5992, "theoretical_loss": 4.16957031509043, "tokens_seen": 303300608 }, { "epoch": 0.09, "learning_rate": 0.0007338014042126379, "loss": 1.601, "theoretical_loss": 4.169465528716295, "tokens_seen": 303366144 }, { "epoch": 0.09, "learning_rate": 0.0007337853560682047, "loss": 1.4711, "theoretical_loss": 4.169360771313434, "tokens_seen": 303431680 }, { "epoch": 0.09, "learning_rate": 0.0007337693079237713, "loss": 1.7924, "theoretical_loss": 4.169256042867579, "tokens_seen": 303497216 }, { "epoch": 0.09, "learning_rate": 0.0007337532597793381, "loss": 1.6994, "theoretical_loss": 4.169151343364476, "tokens_seen": 303562752 }, { "epoch": 0.09, "learning_rate": 0.0007337372116349047, "loss": 1.5778, "theoretical_loss": 4.169046672789882, "tokens_seen": 303628288 }, { "epoch": 0.09, "learning_rate": 0.0007337211634904715, "loss": 1.6038, "theoretical_loss": 4.168942031129562, "tokens_seen": 303693824 }, { "epoch": 0.09, "learning_rate": 0.0007337051153460381, "loss": 1.6144, "theoretical_loss": 4.168837418369292, "tokens_seen": 303759360 }, { "epoch": 0.09, "learning_rate": 0.0007336890672016049, "loss": 1.6138, "theoretical_loss": 4.168732834494857, "tokens_seen": 303824896 }, { "epoch": 0.09, "learning_rate": 0.0007336730190571715, "loss": 1.5502, "theoretical_loss": 4.1686282794920535, "tokens_seen": 303890432 }, { "epoch": 0.09, "learning_rate": 0.0007336569709127383, "loss": 1.7256, "theoretical_loss": 4.168523753346687, "tokens_seen": 303955968 }, { "epoch": 0.09, "learning_rate": 0.0007336409227683049, "loss": 1.5377, "theoretical_loss": 4.168419256044573, "tokens_seen": 304021504 }, { "epoch": 0.09, "learning_rate": 0.0007336248746238717, "loss": 1.632, "theoretical_loss": 4.168314787571538, "tokens_seen": 304087040 }, { "epoch": 0.09, "learning_rate": 0.0007336088264794383, "loss": 1.626, "theoretical_loss": 4.168210347913418, "tokens_seen": 304152576 }, { "epoch": 0.09, "learning_rate": 0.0007335927783350051, "loss": 1.6017, "theoretical_loss": 4.168105937056059, "tokens_seen": 304218112 }, { "epoch": 0.09, "learning_rate": 0.0007335767301905717, "loss": 1.7149, "theoretical_loss": 4.168001554985317, "tokens_seen": 304283648 }, { "epoch": 0.09, "learning_rate": 0.0007335606820461385, "loss": 1.6081, "theoretical_loss": 4.167897201687056, "tokens_seen": 304349184 }, { "epoch": 0.09, "learning_rate": 0.0007335446339017051, "loss": 1.6175, "theoretical_loss": 4.167792877147155, "tokens_seen": 304414720 }, { "epoch": 0.09, "learning_rate": 0.0007335285857572719, "loss": 1.5451, "theoretical_loss": 4.1676885813514986, "tokens_seen": 304480256 }, { "epoch": 0.09, "learning_rate": 0.0007335125376128385, "loss": 1.6038, "theoretical_loss": 4.167584314285982, "tokens_seen": 304545792 }, { "epoch": 0.09, "learning_rate": 0.0007334964894684053, "loss": 1.5534, "theoretical_loss": 4.167480075936512, "tokens_seen": 304611328 }, { "epoch": 0.09, "learning_rate": 0.000733480441323972, "loss": 1.5889, "theoretical_loss": 4.167375866289004, "tokens_seen": 304676864 }, { "epoch": 0.09, "objective/train/docs_used": 107694, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6512261629104614, "objective/train/theoretical_loss": 4.167271685329383, "objective/train/tokens_used": 325202400, "theoretical_loss": 4.167271685329383, "tokens_seen": 304742400 }, { "epoch": 0.09, "learning_rate": 0.0007334643931795387, "loss": 1.5713, "theoretical_loss": 4.167271685329383, "tokens_seen": 304742400 }, { "epoch": 0.09, "learning_rate": 0.0007334483450351053, "loss": 1.6476, "theoretical_loss": 4.167167533043585, "tokens_seen": 304807936 }, { "epoch": 0.09, "learning_rate": 0.0007334322968906721, "loss": 1.63, "theoretical_loss": 4.1670634094175565, "tokens_seen": 304873472 }, { "epoch": 0.09, "learning_rate": 0.0007334162487462387, "loss": 1.541, "theoretical_loss": 4.166959314437252, "tokens_seen": 304939008 }, { "epoch": 0.09, "learning_rate": 0.0007334002006018055, "loss": 1.6109, "theoretical_loss": 4.166855248088637, "tokens_seen": 305004544 }, { "epoch": 0.09, "learning_rate": 0.0007333841524573721, "loss": 1.6243, "theoretical_loss": 4.1667512103576865, "tokens_seen": 305070080 }, { "epoch": 0.09, "learning_rate": 0.0007333681043129389, "loss": 1.6751, "theoretical_loss": 4.166647201230386, "tokens_seen": 305135616 }, { "epoch": 0.09, "learning_rate": 0.0007333520561685056, "loss": 1.6589, "theoretical_loss": 4.166543220692731, "tokens_seen": 305201152 }, { "epoch": 0.09, "learning_rate": 0.0007333360080240723, "loss": 1.5901, "theoretical_loss": 4.166439268730724, "tokens_seen": 305266688 }, { "epoch": 0.09, "learning_rate": 0.000733319959879639, "loss": 1.653, "theoretical_loss": 4.166335345330383, "tokens_seen": 305332224 }, { "epoch": 0.09, "learning_rate": 0.0007333039117352057, "loss": 1.6598, "theoretical_loss": 4.16623145047773, "tokens_seen": 305397760 }, { "epoch": 0.09, "learning_rate": 0.0007332878635907724, "loss": 1.5896, "theoretical_loss": 4.166127584158801, "tokens_seen": 305463296 }, { "epoch": 0.09, "learning_rate": 0.0007332718154463391, "loss": 1.6481, "theoretical_loss": 4.166023746359639, "tokens_seen": 305528832 }, { "epoch": 0.09, "learning_rate": 0.0007332557673019058, "loss": 1.6581, "theoretical_loss": 4.165919937066299, "tokens_seen": 305594368 }, { "epoch": 0.09, "learning_rate": 0.0007332397191574725, "loss": 1.6001, "theoretical_loss": 4.165816156264844, "tokens_seen": 305659904 }, { "epoch": 0.09, "learning_rate": 0.0007332236710130392, "loss": 1.7288, "theoretical_loss": 4.165712403941348, "tokens_seen": 305725440 }, { "epoch": 0.09, "learning_rate": 0.0007332076228686059, "loss": 1.5835, "theoretical_loss": 4.1656086800818946, "tokens_seen": 305790976 }, { "epoch": 0.09, "learning_rate": 0.0007331915747241726, "loss": 1.6093, "theoretical_loss": 4.165504984672577, "tokens_seen": 305856512 }, { "epoch": 0.09, "learning_rate": 0.0007331755265797393, "loss": 1.4599, "theoretical_loss": 4.165401317699498, "tokens_seen": 305922048 }, { "epoch": 0.09, "learning_rate": 0.000733159478435306, "loss": 1.6132, "theoretical_loss": 4.16529767914877, "tokens_seen": 305987584 }, { "epoch": 0.09, "learning_rate": 0.0007331434302908727, "loss": 1.5185, "theoretical_loss": 4.165194069006516, "tokens_seen": 306053120 }, { "epoch": 0.09, "learning_rate": 0.0007331273821464394, "loss": 1.6518, "theoretical_loss": 4.165090487258867, "tokens_seen": 306118656 }, { "epoch": 0.09, "learning_rate": 0.0007331113340020061, "loss": 1.5807, "theoretical_loss": 4.164986933891968, "tokens_seen": 306184192 }, { "epoch": 0.09, "learning_rate": 0.0007330952858575728, "loss": 1.6028, "theoretical_loss": 4.164883408891968, "tokens_seen": 306249728 }, { "epoch": 0.09, "learning_rate": 0.0007330792377131395, "loss": 1.6098, "theoretical_loss": 4.164779912245029, "tokens_seen": 306315264 }, { "epoch": 0.09, "learning_rate": 0.0007330631895687062, "loss": 1.5847, "theoretical_loss": 4.1646764439373225, "tokens_seen": 306380800 }, { "epoch": 0.09, "learning_rate": 0.0007330471414242729, "loss": 1.5105, "theoretical_loss": 4.164573003955029, "tokens_seen": 306446336 }, { "epoch": 0.09, "learning_rate": 0.0007330310932798396, "loss": 1.5924, "theoretical_loss": 4.164469592284338, "tokens_seen": 306511872 }, { "epoch": 0.09, "learning_rate": 0.0007330150451354062, "loss": 1.7551, "theoretical_loss": 4.164366208911453, "tokens_seen": 306577408 }, { "epoch": 0.09, "learning_rate": 0.000732998996990973, "loss": 1.7007, "theoretical_loss": 4.1642628538225805, "tokens_seen": 306642944 }, { "epoch": 0.09, "learning_rate": 0.0007329829488465396, "loss": 1.6358, "theoretical_loss": 4.1641595270039415, "tokens_seen": 306708480 }, { "epoch": 0.09, "learning_rate": 0.0007329669007021064, "loss": 1.6819, "theoretical_loss": 4.1640562284417655, "tokens_seen": 306774016 }, { "epoch": 0.09, "learning_rate": 0.000732950852557673, "loss": 1.5444, "theoretical_loss": 4.16395295812229, "tokens_seen": 306839552 }, { "epoch": 0.09, "learning_rate": 0.0007329348044132398, "loss": 1.6675, "theoretical_loss": 4.1638497160317645, "tokens_seen": 306905088 }, { "epoch": 0.09, "learning_rate": 0.0007329187562688064, "loss": 1.6466, "theoretical_loss": 4.163746502156448, "tokens_seen": 306970624 }, { "epoch": 0.09, "learning_rate": 0.0007329027081243732, "loss": 1.4845, "theoretical_loss": 4.163643316482606, "tokens_seen": 307036160 }, { "epoch": 0.09, "learning_rate": 0.0007328866599799398, "loss": 1.6061, "theoretical_loss": 4.1635401589965175, "tokens_seen": 307101696 }, { "epoch": 0.09, "learning_rate": 0.0007328706118355065, "loss": 1.6429, "theoretical_loss": 4.163437029684469, "tokens_seen": 307167232 }, { "epoch": 0.09, "learning_rate": 0.0007328545636910732, "loss": 1.6264, "theoretical_loss": 4.163333928532758, "tokens_seen": 307232768 }, { "epoch": 0.09, "learning_rate": 0.0007328385155466399, "loss": 1.6924, "theoretical_loss": 4.163230855527688, "tokens_seen": 307298304 }, { "epoch": 0.09, "learning_rate": 0.0007328224674022066, "loss": 1.6224, "theoretical_loss": 4.163127810655578, "tokens_seen": 307363840 }, { "epoch": 0.09, "learning_rate": 0.0007328064192577733, "loss": 1.6408, "theoretical_loss": 4.163024793902752, "tokens_seen": 307429376 }, { "epoch": 0.09, "learning_rate": 0.00073279037111334, "loss": 1.7319, "theoretical_loss": 4.162921805255543, "tokens_seen": 307494912 }, { "epoch": 0.09, "learning_rate": 0.0007327743229689067, "loss": 1.501, "theoretical_loss": 4.162818844700298, "tokens_seen": 307560448 }, { "epoch": 0.09, "learning_rate": 0.0007327582748244734, "loss": 1.6282, "theoretical_loss": 4.162715912223369, "tokens_seen": 307625984 }, { "epoch": 0.09, "learning_rate": 0.0007327422266800401, "loss": 1.5315, "theoretical_loss": 4.162613007811122, "tokens_seen": 307691520 }, { "epoch": 0.09, "learning_rate": 0.0007327261785356068, "loss": 1.5266, "theoretical_loss": 4.1625101314499275, "tokens_seen": 307757056 }, { "epoch": 0.09, "learning_rate": 0.0007327101303911736, "loss": 1.5706, "theoretical_loss": 4.162407283126169, "tokens_seen": 307822592 }, { "epoch": 0.09, "learning_rate": 0.0007326940822467402, "loss": 1.5297, "theoretical_loss": 4.1623044628262384, "tokens_seen": 307888128 }, { "epoch": 0.09, "learning_rate": 0.000732678034102307, "loss": 1.6028, "theoretical_loss": 4.162201670536537, "tokens_seen": 307953664 }, { "epoch": 0.09, "objective/train/docs_used": 108952, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5811710357666016, "objective/train/theoretical_loss": 4.162098906243477, "objective/train/tokens_used": 328479200, "theoretical_loss": 4.162098906243477, "tokens_seen": 308019200 }, { "epoch": 0.09, "learning_rate": 0.0007326619859578736, "loss": 1.6008, "theoretical_loss": 4.162098906243477, "tokens_seen": 308019200 }, { "epoch": 0.09, "learning_rate": 0.0007326459378134404, "loss": 1.5509, "theoretical_loss": 4.161996169933477, "tokens_seen": 308084736 }, { "epoch": 0.09, "learning_rate": 0.000732629889669007, "loss": 1.6285, "theoretical_loss": 4.161893461592968, "tokens_seen": 308150272 }, { "epoch": 0.09, "learning_rate": 0.0007326138415245738, "loss": 1.6352, "theoretical_loss": 4.16179078120839, "tokens_seen": 308215808 }, { "epoch": 0.09, "learning_rate": 0.0007325977933801404, "loss": 1.5552, "theoretical_loss": 4.161688128766191, "tokens_seen": 308281344 }, { "epoch": 0.09, "learning_rate": 0.0007325817452357072, "loss": 1.5523, "theoretical_loss": 4.16158550425283, "tokens_seen": 308346880 }, { "epoch": 0.09, "learning_rate": 0.0007325656970912738, "loss": 1.6311, "theoretical_loss": 4.161482907654775, "tokens_seen": 308412416 }, { "epoch": 0.09, "learning_rate": 0.0007325496489468406, "loss": 1.6177, "theoretical_loss": 4.161380338958502, "tokens_seen": 308477952 }, { "epoch": 0.09, "learning_rate": 0.0007325336008024072, "loss": 1.6076, "theoretical_loss": 4.161277798150498, "tokens_seen": 308543488 }, { "epoch": 0.09, "learning_rate": 0.000732517552657974, "loss": 1.5029, "theoretical_loss": 4.16117528521726, "tokens_seen": 308609024 }, { "epoch": 0.09, "learning_rate": 0.0007325015045135406, "loss": 1.6536, "theoretical_loss": 4.161072800145292, "tokens_seen": 308674560 }, { "epoch": 0.09, "learning_rate": 0.0007324854563691074, "loss": 1.5278, "theoretical_loss": 4.16097034292111, "tokens_seen": 308740096 }, { "epoch": 0.09, "learning_rate": 0.000732469408224674, "loss": 1.4431, "theoretical_loss": 4.160867913531238, "tokens_seen": 308805632 }, { "epoch": 0.09, "learning_rate": 0.0007324533600802408, "loss": 1.6229, "theoretical_loss": 4.160765511962209, "tokens_seen": 308871168 }, { "epoch": 0.09, "learning_rate": 0.0007324373119358074, "loss": 1.5469, "theoretical_loss": 4.160663138200567, "tokens_seen": 308936704 }, { "epoch": 0.09, "learning_rate": 0.0007324212637913742, "loss": 1.6081, "theoretical_loss": 4.1605607922328645, "tokens_seen": 309002240 }, { "epoch": 0.09, "learning_rate": 0.0007324052156469408, "loss": 1.6369, "theoretical_loss": 4.160458474045662, "tokens_seen": 309067776 }, { "epoch": 0.09, "learning_rate": 0.0007323891675025076, "loss": 1.736, "theoretical_loss": 4.16035618362553, "tokens_seen": 309133312 }, { "epoch": 0.09, "learning_rate": 0.0007323731193580742, "loss": 1.4865, "theoretical_loss": 4.16025392095905, "tokens_seen": 309198848 }, { "epoch": 0.09, "learning_rate": 0.000732357071213641, "loss": 1.6449, "theoretical_loss": 4.160151686032814, "tokens_seen": 309264384 }, { "epoch": 0.09, "learning_rate": 0.0007323410230692076, "loss": 1.622, "theoretical_loss": 4.160049478833416, "tokens_seen": 309329920 }, { "epoch": 0.09, "learning_rate": 0.0007323249749247744, "loss": 1.681, "theoretical_loss": 4.159947299347468, "tokens_seen": 309395456 }, { "epoch": 0.09, "learning_rate": 0.000732308926780341, "loss": 1.5762, "theoretical_loss": 4.159845147561587, "tokens_seen": 309460992 }, { "epoch": 0.09, "learning_rate": 0.0007322928786359078, "loss": 1.5544, "theoretical_loss": 4.1597430234624, "tokens_seen": 309526528 }, { "epoch": 0.09, "learning_rate": 0.0007322768304914744, "loss": 1.4294, "theoretical_loss": 4.159640927036541, "tokens_seen": 309592064 }, { "epoch": 0.09, "learning_rate": 0.0007322607823470412, "loss": 1.5333, "theoretical_loss": 4.159538858270659, "tokens_seen": 309657600 }, { "epoch": 0.09, "learning_rate": 0.0007322447342026078, "loss": 1.5659, "theoretical_loss": 4.159436817151407, "tokens_seen": 309723136 }, { "epoch": 0.09, "learning_rate": 0.0007322286860581746, "loss": 1.6312, "theoretical_loss": 4.159334803665448, "tokens_seen": 309788672 }, { "epoch": 0.09, "learning_rate": 0.0007322126379137412, "loss": 1.5842, "theoretical_loss": 4.159232817799458, "tokens_seen": 309854208 }, { "epoch": 0.09, "learning_rate": 0.000732196589769308, "loss": 1.5808, "theoretical_loss": 4.1591308595401175, "tokens_seen": 309919744 }, { "epoch": 0.09, "learning_rate": 0.0007321805416248746, "loss": 1.5817, "theoretical_loss": 4.159028928874118, "tokens_seen": 309985280 }, { "epoch": 0.09, "learning_rate": 0.0007321644934804414, "loss": 1.6191, "theoretical_loss": 4.158927025788162, "tokens_seen": 310050816 }, { "epoch": 0.09, "learning_rate": 0.000732148445336008, "loss": 1.624, "theoretical_loss": 4.158825150268958, "tokens_seen": 310116352 }, { "epoch": 0.09, "learning_rate": 0.0007321323971915748, "loss": 1.5717, "theoretical_loss": 4.158723302303227, "tokens_seen": 310181888 }, { "epoch": 0.09, "learning_rate": 0.0007321163490471414, "loss": 1.6036, "theoretical_loss": 4.158621481877697, "tokens_seen": 310247424 }, { "epoch": 0.09, "learning_rate": 0.0007321003009027082, "loss": 1.6511, "theoretical_loss": 4.158519688979106, "tokens_seen": 310312960 }, { "epoch": 0.09, "learning_rate": 0.0007320842527582748, "loss": 1.5586, "theoretical_loss": 4.1584179235942, "tokens_seen": 310378496 }, { "epoch": 0.09, "learning_rate": 0.0007320682046138416, "loss": 1.6026, "theoretical_loss": 4.158316185709737, "tokens_seen": 310444032 }, { "epoch": 0.09, "learning_rate": 0.0007320521564694082, "loss": 1.7038, "theoretical_loss": 4.158214475312481, "tokens_seen": 310509568 }, { "epoch": 0.09, "learning_rate": 0.000732036108324975, "loss": 1.7325, "theoretical_loss": 4.158112792389206, "tokens_seen": 310575104 }, { "epoch": 0.09, "learning_rate": 0.0007320200601805416, "loss": 1.6455, "theoretical_loss": 4.158011136926698, "tokens_seen": 310640640 }, { "epoch": 0.09, "learning_rate": 0.0007320040120361084, "loss": 1.5333, "theoretical_loss": 4.157909508911747, "tokens_seen": 310706176 }, { "epoch": 0.09, "learning_rate": 0.000731987963891675, "loss": 1.663, "theoretical_loss": 4.157807908331157, "tokens_seen": 310771712 }, { "epoch": 0.09, "learning_rate": 0.0007319719157472418, "loss": 1.6373, "theoretical_loss": 4.157706335171738, "tokens_seen": 310837248 }, { "epoch": 0.09, "learning_rate": 0.0007319558676028085, "loss": 1.6038, "theoretical_loss": 4.15760478942031, "tokens_seen": 310902784 }, { "epoch": 0.09, "learning_rate": 0.0007319398194583752, "loss": 1.6834, "theoretical_loss": 4.157503271063703, "tokens_seen": 310968320 }, { "epoch": 0.09, "learning_rate": 0.0007319237713139419, "loss": 1.529, "theoretical_loss": 4.157401780088756, "tokens_seen": 311033856 }, { "epoch": 0.09, "learning_rate": 0.0007319077231695086, "loss": 1.671, "theoretical_loss": 4.157300316482315, "tokens_seen": 311099392 }, { "epoch": 0.09, "learning_rate": 0.0007318916750250753, "loss": 1.6511, "theoretical_loss": 4.157198880231238, "tokens_seen": 311164928 }, { "epoch": 0.09, "learning_rate": 0.000731875626880642, "loss": 1.5657, "theoretical_loss": 4.157097471322389, "tokens_seen": 311230464 }, { "epoch": 0.09, "objective/train/docs_used": 110010, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5765893459320068, "objective/train/theoretical_loss": 4.156996089742645, "objective/train/tokens_used": 331756000, "theoretical_loss": 4.156996089742645, "tokens_seen": 311296000 }, { "epoch": 0.09, "learning_rate": 0.0007318595787362087, "loss": 1.56, "theoretical_loss": 4.156996089742645, "tokens_seen": 311296000 }, { "epoch": 0.09, "learning_rate": 0.0007318435305917754, "loss": 1.7442, "theoretical_loss": 4.156894735478888, "tokens_seen": 311361536 }, { "epoch": 0.09, "learning_rate": 0.0007318274824473421, "loss": 1.5361, "theoretical_loss": 4.156793408518011, "tokens_seen": 311427072 }, { "epoch": 0.09, "learning_rate": 0.0007318114343029088, "loss": 1.6328, "theoretical_loss": 4.156692108846916, "tokens_seen": 311492608 }, { "epoch": 0.09, "learning_rate": 0.0007317953861584755, "loss": 1.6673, "theoretical_loss": 4.156590836452514, "tokens_seen": 311558144 }, { "epoch": 0.09, "learning_rate": 0.0007317793380140422, "loss": 1.7033, "theoretical_loss": 4.156489591321726, "tokens_seen": 311623680 }, { "epoch": 0.09, "learning_rate": 0.0007317632898696089, "loss": 1.4632, "theoretical_loss": 4.15638837344148, "tokens_seen": 311689216 }, { "epoch": 0.09, "learning_rate": 0.0007317472417251756, "loss": 1.6694, "theoretical_loss": 4.156287182798714, "tokens_seen": 311754752 }, { "epoch": 0.09, "learning_rate": 0.0007317311935807423, "loss": 1.552, "theoretical_loss": 4.156186019380375, "tokens_seen": 311820288 }, { "epoch": 0.09, "learning_rate": 0.000731715145436309, "loss": 1.6538, "theoretical_loss": 4.156084883173419, "tokens_seen": 311885824 }, { "epoch": 0.09, "learning_rate": 0.0007316990972918757, "loss": 1.5271, "theoretical_loss": 4.155983774164811, "tokens_seen": 311951360 }, { "epoch": 0.09, "learning_rate": 0.0007316830491474424, "loss": 1.4608, "theoretical_loss": 4.1558826923415255, "tokens_seen": 312016896 }, { "epoch": 0.09, "learning_rate": 0.0007316670010030091, "loss": 1.6252, "theoretical_loss": 4.155781637690545, "tokens_seen": 312082432 }, { "epoch": 0.09, "learning_rate": 0.0007316509528585758, "loss": 1.5328, "theoretical_loss": 4.1556806101988615, "tokens_seen": 312147968 }, { "epoch": 0.09, "learning_rate": 0.0007316349047141425, "loss": 1.5291, "theoretical_loss": 4.155579609853476, "tokens_seen": 312213504 }, { "epoch": 0.09, "learning_rate": 0.0007316188565697092, "loss": 1.5796, "theoretical_loss": 4.155478636641398, "tokens_seen": 312279040 }, { "epoch": 0.09, "learning_rate": 0.0007316028084252759, "loss": 1.7283, "theoretical_loss": 4.155377690549646, "tokens_seen": 312344576 }, { "epoch": 0.09, "learning_rate": 0.0007315867602808425, "loss": 1.5384, "theoretical_loss": 4.155276771565248, "tokens_seen": 312410112 }, { "epoch": 0.09, "learning_rate": 0.0007315707121364093, "loss": 1.6876, "theoretical_loss": 4.155175879675241, "tokens_seen": 312475648 }, { "epoch": 0.09, "learning_rate": 0.0007315546639919759, "loss": 1.5711, "theoretical_loss": 4.155075014866671, "tokens_seen": 312541184 }, { "epoch": 0.09, "learning_rate": 0.0007315386158475427, "loss": 1.6038, "theoretical_loss": 4.15497417712659, "tokens_seen": 312606720 }, { "epoch": 0.09, "learning_rate": 0.0007315225677031093, "loss": 1.6181, "theoretical_loss": 4.1548733664420645, "tokens_seen": 312672256 }, { "epoch": 0.09, "learning_rate": 0.0007315065195586761, "loss": 1.6023, "theoretical_loss": 4.154772582800165, "tokens_seen": 312737792 }, { "epoch": 0.09, "learning_rate": 0.0007314904714142427, "loss": 1.6421, "theoretical_loss": 4.154671826187972, "tokens_seen": 312803328 }, { "epoch": 0.09, "learning_rate": 0.0007314744232698095, "loss": 1.6457, "theoretical_loss": 4.154571096592576, "tokens_seen": 312868864 }, { "epoch": 0.09, "learning_rate": 0.0007314583751253761, "loss": 1.6198, "theoretical_loss": 4.154470394001077, "tokens_seen": 312934400 }, { "epoch": 0.09, "learning_rate": 0.0007314423269809429, "loss": 1.5943, "theoretical_loss": 4.154369718400583, "tokens_seen": 312999936 }, { "epoch": 0.09, "learning_rate": 0.0007314262788365095, "loss": 1.5763, "theoretical_loss": 4.154269069778207, "tokens_seen": 313065472 }, { "epoch": 0.09, "learning_rate": 0.0007314102306920763, "loss": 1.6197, "theoretical_loss": 4.154168448121078, "tokens_seen": 313131008 }, { "epoch": 0.09, "learning_rate": 0.0007313941825476429, "loss": 1.6308, "theoretical_loss": 4.15406785341633, "tokens_seen": 313196544 }, { "epoch": 0.09, "learning_rate": 0.0007313781344032097, "loss": 1.488, "theoretical_loss": 4.153967285651105, "tokens_seen": 313262080 }, { "epoch": 0.09, "learning_rate": 0.0007313620862587763, "loss": 1.7241, "theoretical_loss": 4.153866744812555, "tokens_seen": 313327616 }, { "epoch": 0.09, "learning_rate": 0.0007313460381143431, "loss": 1.488, "theoretical_loss": 4.1537662308878405, "tokens_seen": 313393152 }, { "epoch": 0.09, "learning_rate": 0.0007313299899699097, "loss": 1.685, "theoretical_loss": 4.153665743864131, "tokens_seen": 313458688 }, { "epoch": 0.1, "learning_rate": 0.0007313139418254765, "loss": 1.5907, "theoretical_loss": 4.153565283728606, "tokens_seen": 313524224 }, { "epoch": 0.1, "learning_rate": 0.0007312978936810431, "loss": 1.6424, "theoretical_loss": 4.153464850468452, "tokens_seen": 313589760 }, { "epoch": 0.1, "learning_rate": 0.0007312818455366099, "loss": 1.6169, "theoretical_loss": 4.1533644440708635, "tokens_seen": 313655296 }, { "epoch": 0.1, "learning_rate": 0.0007312657973921765, "loss": 1.6373, "theoretical_loss": 4.153264064523047, "tokens_seen": 313720832 }, { "epoch": 0.1, "learning_rate": 0.0007312497492477433, "loss": 1.4888, "theoretical_loss": 4.1531637118122156, "tokens_seen": 313786368 }, { "epoch": 0.1, "learning_rate": 0.0007312337011033099, "loss": 1.6178, "theoretical_loss": 4.153063385925591, "tokens_seen": 313851904 }, { "epoch": 0.1, "learning_rate": 0.0007312176529588767, "loss": 1.6062, "theoretical_loss": 4.152963086850405, "tokens_seen": 313917440 }, { "epoch": 0.1, "learning_rate": 0.0007312016048144433, "loss": 1.7214, "theoretical_loss": 4.152862814573895, "tokens_seen": 313982976 }, { "epoch": 0.1, "learning_rate": 0.0007311855566700101, "loss": 1.5936, "theoretical_loss": 4.152762569083313, "tokens_seen": 314048512 }, { "epoch": 0.1, "learning_rate": 0.0007311695085255767, "loss": 1.6886, "theoretical_loss": 4.1526623503659135, "tokens_seen": 314114048 }, { "epoch": 0.1, "learning_rate": 0.0007311534603811435, "loss": 1.543, "theoretical_loss": 4.152562158408962, "tokens_seen": 314179584 }, { "epoch": 0.1, "learning_rate": 0.0007311374122367101, "loss": 1.5865, "theoretical_loss": 4.1524619931997355, "tokens_seen": 314245120 }, { "epoch": 0.1, "learning_rate": 0.0007311213640922769, "loss": 1.5747, "theoretical_loss": 4.152361854725515, "tokens_seen": 314310656 }, { "epoch": 0.1, "learning_rate": 0.0007311053159478435, "loss": 1.6538, "theoretical_loss": 4.152261742973594, "tokens_seen": 314376192 }, { "epoch": 0.1, "learning_rate": 0.0007310892678034103, "loss": 1.6593, "theoretical_loss": 4.152161657931273, "tokens_seen": 314441728 }, { "epoch": 0.1, "learning_rate": 0.0007310732196589769, "loss": 1.6103, "theoretical_loss": 4.15206159958586, "tokens_seen": 314507264 }, { "epoch": 0.1, "objective/train/docs_used": 111213, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5043652057647705, "objective/train/theoretical_loss": 4.1519615679246735, "objective/train/tokens_used": 335032800, "theoretical_loss": 4.1519615679246735, "tokens_seen": 314572800 }, { "epoch": 0.1, "learning_rate": 0.0007310571715145437, "loss": 1.5245, "theoretical_loss": 4.1519615679246735, "tokens_seen": 314572800 }, { "epoch": 0.1, "learning_rate": 0.0007310411233701103, "loss": 1.5779, "theoretical_loss": 4.151861562935041, "tokens_seen": 314638336 }, { "epoch": 0.1, "learning_rate": 0.0007310250752256771, "loss": 1.5589, "theoretical_loss": 4.151761584604298, "tokens_seen": 314703872 }, { "epoch": 0.1, "learning_rate": 0.0007310090270812437, "loss": 1.5793, "theoretical_loss": 4.1516616329197875, "tokens_seen": 314769408 }, { "epoch": 0.1, "learning_rate": 0.0007309929789368105, "loss": 1.6619, "theoretical_loss": 4.151561707868863, "tokens_seen": 314834944 }, { "epoch": 0.1, "learning_rate": 0.0007309769307923771, "loss": 1.6171, "theoretical_loss": 4.151461809438885, "tokens_seen": 314900480 }, { "epoch": 0.1, "learning_rate": 0.0007309608826479439, "loss": 1.5683, "theoretical_loss": 4.1513619376172235, "tokens_seen": 314966016 }, { "epoch": 0.1, "learning_rate": 0.0007309448345035105, "loss": 1.6547, "theoretical_loss": 4.151262092391257, "tokens_seen": 315031552 }, { "epoch": 0.1, "learning_rate": 0.0007309287863590773, "loss": 1.6485, "theoretical_loss": 4.1511622737483735, "tokens_seen": 315097088 }, { "epoch": 0.1, "learning_rate": 0.0007309127382146439, "loss": 1.5209, "theoretical_loss": 4.151062481675968, "tokens_seen": 315162624 }, { "epoch": 0.1, "learning_rate": 0.0007308966900702107, "loss": 1.5688, "theoretical_loss": 4.150962716161445, "tokens_seen": 315228160 }, { "epoch": 0.1, "learning_rate": 0.0007308806419257773, "loss": 1.5452, "theoretical_loss": 4.150862977192217, "tokens_seen": 315293696 }, { "epoch": 0.1, "learning_rate": 0.0007308645937813441, "loss": 1.555, "theoretical_loss": 4.150763264755706, "tokens_seen": 315359232 }, { "epoch": 0.1, "learning_rate": 0.0007308485456369107, "loss": 1.7569, "theoretical_loss": 4.150663578839342, "tokens_seen": 315424768 }, { "epoch": 0.1, "learning_rate": 0.0007308324974924775, "loss": 1.598, "theoretical_loss": 4.150563919430562, "tokens_seen": 315490304 }, { "epoch": 0.1, "learning_rate": 0.0007308164493480441, "loss": 1.4641, "theoretical_loss": 4.150464286516816, "tokens_seen": 315555840 }, { "epoch": 0.1, "learning_rate": 0.0007308004012036109, "loss": 1.5865, "theoretical_loss": 4.150364680085558, "tokens_seen": 315621376 }, { "epoch": 0.1, "learning_rate": 0.0007307843530591775, "loss": 1.5584, "theoretical_loss": 4.150265100124253, "tokens_seen": 315686912 }, { "epoch": 0.1, "learning_rate": 0.0007307683049147443, "loss": 1.6787, "theoretical_loss": 4.150165546620373, "tokens_seen": 315752448 }, { "epoch": 0.1, "learning_rate": 0.0007307522567703109, "loss": 1.7157, "theoretical_loss": 4.1500660195614, "tokens_seen": 315817984 }, { "epoch": 0.1, "learning_rate": 0.0007307362086258777, "loss": 1.5774, "theoretical_loss": 4.149966518934824, "tokens_seen": 315883520 }, { "epoch": 0.1, "learning_rate": 0.0007307201604814443, "loss": 1.5534, "theoretical_loss": 4.149867044728143, "tokens_seen": 315949056 }, { "epoch": 0.1, "learning_rate": 0.0007307041123370111, "loss": 1.597, "theoretical_loss": 4.149767596928863, "tokens_seen": 316014592 }, { "epoch": 0.1, "learning_rate": 0.0007306880641925777, "loss": 1.8193, "theoretical_loss": 4.149668175524502, "tokens_seen": 316080128 }, { "epoch": 0.1, "learning_rate": 0.0007306720160481445, "loss": 1.5885, "theoretical_loss": 4.14956878050258, "tokens_seen": 316145664 }, { "epoch": 0.1, "learning_rate": 0.0007306559679037111, "loss": 1.6594, "theoretical_loss": 4.1494694118506334, "tokens_seen": 316211200 }, { "epoch": 0.1, "learning_rate": 0.0007306399197592779, "loss": 1.6892, "theoretical_loss": 4.1493700695562, "tokens_seen": 316276736 }, { "epoch": 0.1, "learning_rate": 0.0007306238716148446, "loss": 1.5594, "theoretical_loss": 4.14927075360683, "tokens_seen": 316342272 }, { "epoch": 0.1, "learning_rate": 0.0007306078234704113, "loss": 1.6457, "theoretical_loss": 4.149171463990082, "tokens_seen": 316407808 }, { "epoch": 0.1, "learning_rate": 0.000730591775325978, "loss": 1.5986, "theoretical_loss": 4.149072200693521, "tokens_seen": 316473344 }, { "epoch": 0.1, "learning_rate": 0.0007305757271815447, "loss": 1.6219, "theoretical_loss": 4.148972963704722, "tokens_seen": 316538880 }, { "epoch": 0.1, "learning_rate": 0.0007305596790371114, "loss": 1.6861, "theoretical_loss": 4.148873753011269, "tokens_seen": 316604416 }, { "epoch": 0.1, "learning_rate": 0.0007305436308926781, "loss": 1.6092, "theoretical_loss": 4.148774568600752, "tokens_seen": 316669952 }, { "epoch": 0.1, "learning_rate": 0.0007305275827482448, "loss": 1.5524, "theoretical_loss": 4.148675410460772, "tokens_seen": 316735488 }, { "epoch": 0.1, "learning_rate": 0.0007305115346038115, "loss": 1.5586, "theoretical_loss": 4.148576278578937, "tokens_seen": 316801024 }, { "epoch": 0.1, "learning_rate": 0.0007304954864593782, "loss": 1.5904, "theoretical_loss": 4.148477172942863, "tokens_seen": 316866560 }, { "epoch": 0.1, "learning_rate": 0.0007304794383149449, "loss": 1.745, "theoretical_loss": 4.1483780935401775, "tokens_seen": 316932096 }, { "epoch": 0.1, "learning_rate": 0.0007304633901705116, "loss": 1.5839, "theoretical_loss": 4.148279040358511, "tokens_seen": 316997632 }, { "epoch": 0.1, "learning_rate": 0.0007304473420260783, "loss": 1.5903, "theoretical_loss": 4.148180013385507, "tokens_seen": 317063168 }, { "epoch": 0.1, "learning_rate": 0.000730431293881645, "loss": 1.7813, "theoretical_loss": 4.148081012608817, "tokens_seen": 317128704 }, { "epoch": 0.1, "learning_rate": 0.0007304152457372117, "loss": 1.8107, "theoretical_loss": 4.147982038016096, "tokens_seen": 317194240 }, { "epoch": 0.1, "learning_rate": 0.0007303991975927784, "loss": 1.7651, "theoretical_loss": 4.147883089595015, "tokens_seen": 317259776 }, { "epoch": 0.1, "learning_rate": 0.0007303831494483451, "loss": 1.8327, "theoretical_loss": 4.147784167333247, "tokens_seen": 317325312 }, { "epoch": 0.1, "learning_rate": 0.0007303671013039118, "loss": 1.8399, "theoretical_loss": 4.147685271218477, "tokens_seen": 317390848 }, { "epoch": 0.1, "learning_rate": 0.0007303510531594785, "loss": 1.8931, "theoretical_loss": 4.147586401238396, "tokens_seen": 317456384 }, { "epoch": 0.1, "learning_rate": 0.0007303350050150452, "loss": 1.8639, "theoretical_loss": 4.147487557380705, "tokens_seen": 317521920 }, { "epoch": 0.1, "learning_rate": 0.0007303189568706119, "loss": 1.8167, "theoretical_loss": 4.147388739633113, "tokens_seen": 317587456 }, { "epoch": 0.1, "learning_rate": 0.0007303029087261786, "loss": 1.86, "theoretical_loss": 4.147289947983337, "tokens_seen": 317652992 }, { "epoch": 0.1, "learning_rate": 0.0007302868605817453, "loss": 1.9685, "theoretical_loss": 4.1471911824191015, "tokens_seen": 317718528 }, { "epoch": 0.1, "learning_rate": 0.000730270812437312, "loss": 1.7142, "theoretical_loss": 4.147092442928141, "tokens_seen": 317784064 }, { "epoch": 0.1, "objective/train/docs_used": 112065, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.024470567703247, "objective/train/theoretical_loss": 4.146993729498197, "objective/train/tokens_used": 338309600, "theoretical_loss": 4.146993729498197, "tokens_seen": 317849600 }, { "epoch": 0.1, "learning_rate": 0.0007302547642928787, "loss": 1.9022, "theoretical_loss": 4.146993729498197, "tokens_seen": 317849600 }, { "epoch": 0.1, "learning_rate": 0.0007302387161484454, "loss": 1.9061, "theoretical_loss": 4.146895042117021, "tokens_seen": 317915136 }, { "epoch": 0.1, "learning_rate": 0.0007302226680040121, "loss": 1.9165, "theoretical_loss": 4.146796380772369, "tokens_seen": 317980672 }, { "epoch": 0.1, "learning_rate": 0.0007302066198595788, "loss": 1.9143, "theoretical_loss": 4.146697745452011, "tokens_seen": 318046208 }, { "epoch": 0.1, "learning_rate": 0.0007301905717151454, "loss": 1.8095, "theoretical_loss": 4.146599136143719, "tokens_seen": 318111744 }, { "epoch": 0.1, "learning_rate": 0.0007301745235707122, "loss": 1.7185, "theoretical_loss": 4.146500552835278, "tokens_seen": 318177280 }, { "epoch": 0.1, "learning_rate": 0.0007301584754262788, "loss": 1.8265, "theoretical_loss": 4.146401995514479, "tokens_seen": 318242816 }, { "epoch": 0.1, "learning_rate": 0.0007301424272818456, "loss": 1.8853, "theoretical_loss": 4.146303464169123, "tokens_seen": 318308352 }, { "epoch": 0.1, "learning_rate": 0.0007301263791374122, "loss": 1.8499, "theoretical_loss": 4.146204958787017, "tokens_seen": 318373888 }, { "epoch": 0.1, "learning_rate": 0.000730110330992979, "loss": 1.6803, "theoretical_loss": 4.146106479355978, "tokens_seen": 318439424 }, { "epoch": 0.1, "learning_rate": 0.0007300942828485456, "loss": 1.8647, "theoretical_loss": 4.1460080258638285, "tokens_seen": 318504960 }, { "epoch": 0.1, "learning_rate": 0.0007300782347041124, "loss": 1.8665, "theoretical_loss": 4.145909598298404, "tokens_seen": 318570496 }, { "epoch": 0.1, "learning_rate": 0.000730062186559679, "loss": 1.8231, "theoretical_loss": 4.145811196647544, "tokens_seen": 318636032 }, { "epoch": 0.1, "learning_rate": 0.0007300461384152458, "loss": 1.8109, "theoretical_loss": 4.145712820899098, "tokens_seen": 318701568 }, { "epoch": 0.1, "learning_rate": 0.0007300300902708124, "loss": 1.7429, "theoretical_loss": 4.145614471040923, "tokens_seen": 318767104 }, { "epoch": 0.1, "learning_rate": 0.0007300140421263792, "loss": 1.8226, "theoretical_loss": 4.145516147060884, "tokens_seen": 318832640 }, { "epoch": 0.1, "learning_rate": 0.0007299979939819458, "loss": 1.701, "theoretical_loss": 4.145417848946857, "tokens_seen": 318898176 }, { "epoch": 0.1, "learning_rate": 0.0007299819458375126, "loss": 1.8668, "theoretical_loss": 4.145319576686721, "tokens_seen": 318963712 }, { "epoch": 0.1, "learning_rate": 0.0007299658976930792, "loss": 1.6904, "theoretical_loss": 4.1452213302683685, "tokens_seen": 319029248 }, { "epoch": 0.1, "learning_rate": 0.000729949849548646, "loss": 1.9191, "theoretical_loss": 4.145123109679696, "tokens_seen": 319094784 }, { "epoch": 0.1, "learning_rate": 0.0007299338014042126, "loss": 1.9909, "theoretical_loss": 4.1450249149086105, "tokens_seen": 319160320 }, { "epoch": 0.1, "learning_rate": 0.0007299177532597794, "loss": 1.9859, "theoretical_loss": 4.144926745943026, "tokens_seen": 319225856 }, { "epoch": 0.1, "learning_rate": 0.000729901705115346, "loss": 1.8796, "theoretical_loss": 4.1448286027708665, "tokens_seen": 319291392 }, { "epoch": 0.1, "learning_rate": 0.0007298856569709128, "loss": 1.7589, "theoretical_loss": 4.144730485380062, "tokens_seen": 319356928 }, { "epoch": 0.1, "learning_rate": 0.0007298696088264794, "loss": 1.881, "theoretical_loss": 4.144632393758551, "tokens_seen": 319422464 }, { "epoch": 0.1, "learning_rate": 0.0007298535606820462, "loss": 1.9404, "theoretical_loss": 4.144534327894281, "tokens_seen": 319488000 }, { "epoch": 0.1, "learning_rate": 0.0007298375125376128, "loss": 1.8255, "theoretical_loss": 4.144436287775206, "tokens_seen": 319553536 }, { "epoch": 0.1, "learning_rate": 0.0007298214643931796, "loss": 1.8244, "theoretical_loss": 4.144338273389291, "tokens_seen": 319619072 }, { "epoch": 0.1, "learning_rate": 0.0007298054162487462, "loss": 1.7481, "theoretical_loss": 4.144240284724505, "tokens_seen": 319684608 }, { "epoch": 0.1, "learning_rate": 0.000729789368104313, "loss": 1.8876, "theoretical_loss": 4.144142321768831, "tokens_seen": 319750144 }, { "epoch": 0.1, "learning_rate": 0.0007297733199598796, "loss": 1.7792, "theoretical_loss": 4.144044384510253, "tokens_seen": 319815680 }, { "epoch": 0.1, "learning_rate": 0.0007297572718154464, "loss": 1.7007, "theoretical_loss": 4.1439464729367685, "tokens_seen": 319881216 }, { "epoch": 0.1, "learning_rate": 0.000729741223671013, "loss": 1.5911, "theoretical_loss": 4.14384858703638, "tokens_seen": 319946752 }, { "epoch": 0.1, "learning_rate": 0.0007297251755265798, "loss": 1.7342, "theoretical_loss": 4.1437507267971, "tokens_seen": 320012288 }, { "epoch": 0.1, "learning_rate": 0.0007297091273821464, "loss": 1.7468, "theoretical_loss": 4.143652892206948, "tokens_seen": 320077824 }, { "epoch": 0.1, "learning_rate": 0.0007296930792377132, "loss": 1.7422, "theoretical_loss": 4.1435550832539505, "tokens_seen": 320143360 }, { "epoch": 0.1, "learning_rate": 0.0007296770310932798, "loss": 1.6767, "theoretical_loss": 4.143457299926146, "tokens_seen": 320208896 }, { "epoch": 0.1, "learning_rate": 0.0007296609829488466, "loss": 1.7012, "theoretical_loss": 4.143359542211576, "tokens_seen": 320274432 }, { "epoch": 0.1, "learning_rate": 0.0007296449348044132, "loss": 1.6575, "theoretical_loss": 4.143261810098293, "tokens_seen": 320339968 }, { "epoch": 0.1, "learning_rate": 0.00072962888665998, "loss": 1.8258, "theoretical_loss": 4.1431641035743585, "tokens_seen": 320405504 }, { "epoch": 0.1, "learning_rate": 0.0007296128385155466, "loss": 1.7058, "theoretical_loss": 4.143066422627838, "tokens_seen": 320471040 }, { "epoch": 0.1, "learning_rate": 0.0007295967903711134, "loss": 1.7022, "theoretical_loss": 4.142968767246808, "tokens_seen": 320536576 }, { "epoch": 0.1, "learning_rate": 0.00072958074222668, "loss": 1.658, "theoretical_loss": 4.142871137419354, "tokens_seen": 320602112 }, { "epoch": 0.1, "learning_rate": 0.0007295646940822468, "loss": 1.7651, "theoretical_loss": 4.142773533133565, "tokens_seen": 320667648 }, { "epoch": 0.1, "learning_rate": 0.0007295486459378134, "loss": 1.7998, "theoretical_loss": 4.142675954377543, "tokens_seen": 320733184 }, { "epoch": 0.1, "learning_rate": 0.0007295325977933802, "loss": 1.8644, "theoretical_loss": 4.142578401139396, "tokens_seen": 320798720 }, { "epoch": 0.1, "learning_rate": 0.0007295165496489468, "loss": 1.778, "theoretical_loss": 4.142480873407239, "tokens_seen": 320864256 }, { "epoch": 0.1, "learning_rate": 0.0007295005015045136, "loss": 1.7293, "theoretical_loss": 4.1423833711691955, "tokens_seen": 320929792 }, { "epoch": 0.1, "learning_rate": 0.0007294844533600802, "loss": 1.6623, "theoretical_loss": 4.142285894413398, "tokens_seen": 320995328 }, { "epoch": 0.1, "learning_rate": 0.000729468405215647, "loss": 1.768, "theoretical_loss": 4.142188443127985, "tokens_seen": 321060864 }, { "epoch": 0.1, "objective/train/docs_used": 113306, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.5380922555923462, "objective/train/theoretical_loss": 4.142091017301105, "objective/train/tokens_used": 341586400, "theoretical_loss": 4.142091017301105, "tokens_seen": 321126400 }, { "epoch": 0.1, "learning_rate": 0.0007294523570712136, "loss": 1.6093, "theoretical_loss": 4.142091017301105, "tokens_seen": 321126400 }, { "epoch": 0.1, "learning_rate": 0.0007294363089267804, "loss": 1.6369, "theoretical_loss": 4.141993616920914, "tokens_seen": 321191936 }, { "epoch": 0.1, "learning_rate": 0.000729420260782347, "loss": 1.71, "theoretical_loss": 4.141896241975575, "tokens_seen": 321257472 }, { "epoch": 0.1, "learning_rate": 0.0007294042126379138, "loss": 1.4959, "theoretical_loss": 4.141798892453259, "tokens_seen": 321323008 }, { "epoch": 0.1, "learning_rate": 0.0007293881644934804, "loss": 1.7129, "theoretical_loss": 4.141701568342145, "tokens_seen": 321388544 }, { "epoch": 0.1, "learning_rate": 0.0007293721163490472, "loss": 1.6549, "theoretical_loss": 4.141604269630422, "tokens_seen": 321454080 }, { "epoch": 0.1, "learning_rate": 0.0007293560682046138, "loss": 1.5904, "theoretical_loss": 4.1415069963062825, "tokens_seen": 321519616 }, { "epoch": 0.1, "learning_rate": 0.0007293400200601806, "loss": 1.6822, "theoretical_loss": 4.1414097483579315, "tokens_seen": 321585152 }, { "epoch": 0.1, "learning_rate": 0.0007293239719157472, "loss": 1.6719, "theoretical_loss": 4.1413125257735794, "tokens_seen": 321650688 }, { "epoch": 0.1, "learning_rate": 0.000729307923771314, "loss": 1.5757, "theoretical_loss": 4.141215328541445, "tokens_seen": 321716224 }, { "epoch": 0.1, "learning_rate": 0.0007292918756268806, "loss": 1.5815, "theoretical_loss": 4.141118156649753, "tokens_seen": 321781760 }, { "epoch": 0.1, "learning_rate": 0.0007292758274824474, "loss": 1.6802, "theoretical_loss": 4.1410210100867415, "tokens_seen": 321847296 }, { "epoch": 0.1, "learning_rate": 0.000729259779338014, "loss": 1.6105, "theoretical_loss": 4.14092388884065, "tokens_seen": 321912832 }, { "epoch": 0.1, "learning_rate": 0.0007292437311935808, "loss": 1.5999, "theoretical_loss": 4.1408267928997295, "tokens_seen": 321978368 }, { "epoch": 0.1, "learning_rate": 0.0007292276830491475, "loss": 1.6539, "theoretical_loss": 4.140729722252239, "tokens_seen": 322043904 }, { "epoch": 0.1, "learning_rate": 0.0007292116349047142, "loss": 1.6241, "theoretical_loss": 4.140632676886442, "tokens_seen": 322109440 }, { "epoch": 0.1, "learning_rate": 0.0007291955867602809, "loss": 1.6331, "theoretical_loss": 4.140535656790615, "tokens_seen": 322174976 }, { "epoch": 0.1, "learning_rate": 0.0007291795386158476, "loss": 1.6988, "theoretical_loss": 4.140438661953038, "tokens_seen": 322240512 }, { "epoch": 0.1, "learning_rate": 0.0007291634904714143, "loss": 1.4855, "theoretical_loss": 4.140341692362, "tokens_seen": 322306048 }, { "epoch": 0.1, "learning_rate": 0.000729147442326981, "loss": 1.682, "theoretical_loss": 4.140244748005799, "tokens_seen": 322371584 }, { "epoch": 0.1, "learning_rate": 0.0007291313941825477, "loss": 1.7468, "theoretical_loss": 4.14014782887274, "tokens_seen": 322437120 }, { "epoch": 0.1, "learning_rate": 0.0007291153460381144, "loss": 1.6284, "theoretical_loss": 4.140050934951135, "tokens_seen": 322502656 }, { "epoch": 0.1, "learning_rate": 0.0007290992978936812, "loss": 1.6126, "theoretical_loss": 4.139954066229304, "tokens_seen": 322568192 }, { "epoch": 0.1, "learning_rate": 0.0007290832497492478, "loss": 1.6776, "theoretical_loss": 4.139857222695578, "tokens_seen": 322633728 }, { "epoch": 0.1, "learning_rate": 0.0007290672016048146, "loss": 1.6517, "theoretical_loss": 4.139760404338291, "tokens_seen": 322699264 }, { "epoch": 0.1, "learning_rate": 0.0007290511534603812, "loss": 1.6009, "theoretical_loss": 4.139663611145786, "tokens_seen": 322764800 }, { "epoch": 0.1, "learning_rate": 0.000729035105315948, "loss": 1.6975, "theoretical_loss": 4.139566843106417, "tokens_seen": 322830336 }, { "epoch": 0.1, "learning_rate": 0.0007290190571715146, "loss": 1.6493, "theoretical_loss": 4.139470100208542, "tokens_seen": 322895872 }, { "epoch": 0.1, "learning_rate": 0.0007290030090270814, "loss": 1.7209, "theoretical_loss": 4.139373382440528, "tokens_seen": 322961408 }, { "epoch": 0.1, "learning_rate": 0.000728986960882648, "loss": 1.6022, "theoretical_loss": 4.1392766897907505, "tokens_seen": 323026944 }, { "epoch": 0.1, "learning_rate": 0.0007289709127382148, "loss": 1.6735, "theoretical_loss": 4.139180022247592, "tokens_seen": 323092480 }, { "epoch": 0.1, "learning_rate": 0.0007289548645937814, "loss": 1.5358, "theoretical_loss": 4.139083379799443, "tokens_seen": 323158016 }, { "epoch": 0.1, "learning_rate": 0.0007289388164493481, "loss": 1.5424, "theoretical_loss": 4.1389867624347, "tokens_seen": 323223552 }, { "epoch": 0.1, "learning_rate": 0.0007289227683049148, "loss": 1.6485, "theoretical_loss": 4.138890170141771, "tokens_seen": 323289088 }, { "epoch": 0.1, "learning_rate": 0.0007289067201604815, "loss": 1.5949, "theoretical_loss": 4.138793602909068, "tokens_seen": 323354624 }, { "epoch": 0.1, "learning_rate": 0.0007288906720160482, "loss": 1.7056, "theoretical_loss": 4.138697060725013, "tokens_seen": 323420160 }, { "epoch": 0.1, "learning_rate": 0.0007288746238716149, "loss": 1.6545, "theoretical_loss": 4.138600543578034, "tokens_seen": 323485696 }, { "epoch": 0.1, "learning_rate": 0.0007288585757271816, "loss": 1.7005, "theoretical_loss": 4.138504051456568, "tokens_seen": 323551232 }, { "epoch": 0.1, "learning_rate": 0.0007288425275827483, "loss": 1.6943, "theoretical_loss": 4.13840758434906, "tokens_seen": 323616768 }, { "epoch": 0.1, "learning_rate": 0.000728826479438315, "loss": 1.5391, "theoretical_loss": 4.1383111422439605, "tokens_seen": 323682304 }, { "epoch": 0.1, "learning_rate": 0.0007288104312938817, "loss": 1.6891, "theoretical_loss": 4.13821472512973, "tokens_seen": 323747840 }, { "epoch": 0.1, "learning_rate": 0.0007287943831494483, "loss": 1.6866, "theoretical_loss": 4.138118332994837, "tokens_seen": 323813376 }, { "epoch": 0.1, "learning_rate": 0.0007287783350050151, "loss": 1.69, "theoretical_loss": 4.138021965827753, "tokens_seen": 323878912 }, { "epoch": 0.1, "learning_rate": 0.0007287622868605817, "loss": 1.5937, "theoretical_loss": 4.1379256236169635, "tokens_seen": 323944448 }, { "epoch": 0.1, "learning_rate": 0.0007287462387161485, "loss": 1.6, "theoretical_loss": 4.137829306350957, "tokens_seen": 324009984 }, { "epoch": 0.1, "learning_rate": 0.0007287301905717151, "loss": 1.6746, "theoretical_loss": 4.137733014018233, "tokens_seen": 324075520 }, { "epoch": 0.1, "learning_rate": 0.0007287141424272819, "loss": 1.6548, "theoretical_loss": 4.137636746607295, "tokens_seen": 324141056 }, { "epoch": 0.1, "learning_rate": 0.0007286980942828485, "loss": 1.5518, "theoretical_loss": 4.137540504106657, "tokens_seen": 324206592 }, { "epoch": 0.1, "learning_rate": 0.0007286820461384153, "loss": 1.7232, "theoretical_loss": 4.137444286504841, "tokens_seen": 324272128 }, { "epoch": 0.1, "learning_rate": 0.0007286659979939819, "loss": 1.6377, "theoretical_loss": 4.137348093790372, "tokens_seen": 324337664 }, { "epoch": 0.1, "objective/train/docs_used": 114269, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6439099311828613, "objective/train/theoretical_loss": 4.13725192595179, "objective/train/tokens_used": 344863200, "theoretical_loss": 4.13725192595179, "tokens_seen": 324403200 }, { "epoch": 0.1, "learning_rate": 0.0007286499498495487, "loss": 1.5966, "theoretical_loss": 4.13725192595179, "tokens_seen": 324403200 }, { "epoch": 0.1, "learning_rate": 0.0007286339017051153, "loss": 1.7042, "theoretical_loss": 4.1371557829776355, "tokens_seen": 324468736 }, { "epoch": 0.1, "learning_rate": 0.0007286178535606821, "loss": 1.6878, "theoretical_loss": 4.13705966485646, "tokens_seen": 324534272 }, { "epoch": 0.1, "learning_rate": 0.0007286018054162487, "loss": 1.641, "theoretical_loss": 4.136963571576825, "tokens_seen": 324599808 }, { "epoch": 0.1, "learning_rate": 0.0007285857572718155, "loss": 1.6288, "theoretical_loss": 4.136867503127292, "tokens_seen": 324665344 }, { "epoch": 0.1, "learning_rate": 0.0007285697091273821, "loss": 1.5733, "theoretical_loss": 4.136771459496439, "tokens_seen": 324730880 }, { "epoch": 0.1, "learning_rate": 0.0007285536609829489, "loss": 1.5923, "theoretical_loss": 4.136675440672844, "tokens_seen": 324796416 }, { "epoch": 0.1, "learning_rate": 0.0007285376128385155, "loss": 1.7097, "theoretical_loss": 4.136579446645098, "tokens_seen": 324861952 }, { "epoch": 0.1, "learning_rate": 0.0007285215646940823, "loss": 1.6223, "theoretical_loss": 4.136483477401798, "tokens_seen": 324927488 }, { "epoch": 0.1, "learning_rate": 0.0007285055165496489, "loss": 1.6171, "theoretical_loss": 4.136387532931546, "tokens_seen": 324993024 }, { "epoch": 0.1, "learning_rate": 0.0007284894684052157, "loss": 1.555, "theoretical_loss": 4.136291613222955, "tokens_seen": 325058560 }, { "epoch": 0.1, "learning_rate": 0.0007284734202607823, "loss": 1.5637, "theoretical_loss": 4.136195718264644, "tokens_seen": 325124096 }, { "epoch": 0.1, "learning_rate": 0.0007284573721163491, "loss": 1.5865, "theoretical_loss": 4.136099848045239, "tokens_seen": 325189632 }, { "epoch": 0.1, "learning_rate": 0.0007284413239719157, "loss": 1.695, "theoretical_loss": 4.136004002553375, "tokens_seen": 325255168 }, { "epoch": 0.1, "learning_rate": 0.0007284252758274825, "loss": 1.5485, "theoretical_loss": 4.135908181777693, "tokens_seen": 325320704 }, { "epoch": 0.1, "learning_rate": 0.0007284092276830491, "loss": 1.6128, "theoretical_loss": 4.135812385706842, "tokens_seen": 325386240 }, { "epoch": 0.1, "learning_rate": 0.0007283931795386159, "loss": 1.7098, "theoretical_loss": 4.135716614329479, "tokens_seen": 325451776 }, { "epoch": 0.1, "learning_rate": 0.0007283771313941825, "loss": 1.7113, "theoretical_loss": 4.135620867634269, "tokens_seen": 325517312 }, { "epoch": 0.1, "learning_rate": 0.0007283610832497493, "loss": 1.6371, "theoretical_loss": 4.135525145609881, "tokens_seen": 325582848 }, { "epoch": 0.1, "learning_rate": 0.0007283450351053159, "loss": 1.7067, "theoretical_loss": 4.135429448244997, "tokens_seen": 325648384 }, { "epoch": 0.1, "learning_rate": 0.0007283289869608827, "loss": 1.609, "theoretical_loss": 4.135333775528302, "tokens_seen": 325713920 }, { "epoch": 0.1, "learning_rate": 0.0007283129388164493, "loss": 1.4926, "theoretical_loss": 4.13523812744849, "tokens_seen": 325779456 }, { "epoch": 0.1, "learning_rate": 0.0007282968906720161, "loss": 1.7475, "theoretical_loss": 4.135142503994263, "tokens_seen": 325844992 }, { "epoch": 0.1, "learning_rate": 0.0007282808425275827, "loss": 1.539, "theoretical_loss": 4.13504690515433, "tokens_seen": 325910528 }, { "epoch": 0.1, "learning_rate": 0.0007282647943831495, "loss": 1.6961, "theoretical_loss": 4.134951330917408, "tokens_seen": 325976064 }, { "epoch": 0.1, "learning_rate": 0.0007282487462387161, "loss": 1.5638, "theoretical_loss": 4.134855781272218, "tokens_seen": 326041600 }, { "epoch": 0.1, "learning_rate": 0.0007282326980942829, "loss": 1.6864, "theoretical_loss": 4.134760256207495, "tokens_seen": 326107136 }, { "epoch": 0.1, "learning_rate": 0.0007282166499498495, "loss": 1.4862, "theoretical_loss": 4.1346647557119764, "tokens_seen": 326172672 }, { "epoch": 0.1, "learning_rate": 0.0007282006018054163, "loss": 1.6338, "theoretical_loss": 4.134569279774407, "tokens_seen": 326238208 }, { "epoch": 0.1, "learning_rate": 0.0007281845536609829, "loss": 1.6086, "theoretical_loss": 4.134473828383541, "tokens_seen": 326303744 }, { "epoch": 0.1, "learning_rate": 0.0007281685055165497, "loss": 1.4595, "theoretical_loss": 4.13437840152814, "tokens_seen": 326369280 }, { "epoch": 0.1, "learning_rate": 0.0007281524573721163, "loss": 1.6468, "theoretical_loss": 4.134282999196973, "tokens_seen": 326434816 }, { "epoch": 0.1, "learning_rate": 0.0007281364092276831, "loss": 1.6356, "theoretical_loss": 4.134187621378813, "tokens_seen": 326500352 }, { "epoch": 0.1, "learning_rate": 0.0007281203610832497, "loss": 1.546, "theoretical_loss": 4.134092268062446, "tokens_seen": 326565888 }, { "epoch": 0.1, "learning_rate": 0.0007281043129388165, "loss": 1.4938, "theoretical_loss": 4.133996939236661, "tokens_seen": 326631424 }, { "epoch": 0.1, "learning_rate": 0.0007280882647943831, "loss": 1.6343, "theoretical_loss": 4.133901634890256, "tokens_seen": 326696960 }, { "epoch": 0.1, "learning_rate": 0.0007280722166499499, "loss": 1.5978, "theoretical_loss": 4.1338063550120365, "tokens_seen": 326762496 }, { "epoch": 0.1, "learning_rate": 0.0007280561685055165, "loss": 1.7451, "theoretical_loss": 4.133711099590815, "tokens_seen": 326828032 }, { "epoch": 0.1, "learning_rate": 0.0007280401203610833, "loss": 1.6231, "theoretical_loss": 4.133615868615411, "tokens_seen": 326893568 }, { "epoch": 0.1, "learning_rate": 0.0007280240722166499, "loss": 1.6108, "theoretical_loss": 4.133520662074654, "tokens_seen": 326959104 }, { "epoch": 0.1, "learning_rate": 0.0007280080240722167, "loss": 1.6044, "theoretical_loss": 4.133425479957375, "tokens_seen": 327024640 }, { "epoch": 0.1, "learning_rate": 0.0007279919759277833, "loss": 1.6349, "theoretical_loss": 4.133330322252419, "tokens_seen": 327090176 }, { "epoch": 0.1, "learning_rate": 0.0007279759277833501, "loss": 1.6286, "theoretical_loss": 4.1332351889486345, "tokens_seen": 327155712 }, { "epoch": 0.1, "learning_rate": 0.0007279598796389167, "loss": 1.4971, "theoretical_loss": 4.133140080034878, "tokens_seen": 327221248 }, { "epoch": 0.1, "learning_rate": 0.0007279438314944835, "loss": 1.5259, "theoretical_loss": 4.133044995500013, "tokens_seen": 327286784 }, { "epoch": 0.1, "learning_rate": 0.0007279277833500503, "loss": 1.6473, "theoretical_loss": 4.1329499353329116, "tokens_seen": 327352320 }, { "epoch": 0.1, "learning_rate": 0.0007279117352056169, "loss": 1.5083, "theoretical_loss": 4.132854899522453, "tokens_seen": 327417856 }, { "epoch": 0.1, "learning_rate": 0.0007278956870611837, "loss": 1.7212, "theoretical_loss": 4.132759888057521, "tokens_seen": 327483392 }, { "epoch": 0.1, "learning_rate": 0.0007278796389167503, "loss": 1.6026, "theoretical_loss": 4.13266490092701, "tokens_seen": 327548928 }, { "epoch": 0.1, "learning_rate": 0.0007278635907723171, "loss": 1.5403, "theoretical_loss": 4.132569938119821, "tokens_seen": 327614464 }, { "debugging/Compilability": 0.0, "debugging/distinct-1-grams": 0.7371483758339369, "debugging/entropy-1-grams": 5.361149495192262, "debugging/length": 581.1666666666666, "debugging/num_segments": 12, "debugging/score": 0.06659683474910605, "debugging/score_std": 0.015715120837281396, "epoch": 0.1, "objective/train/docs_used": 115553, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 1.6277862787246704, "objective/train/theoretical_loss": 4.132474999624861, "objective/train/tokens_used": 348140000, "theoretical_loss": 4.132474999624861, "tokens_seen": 327680000 }, { "epoch": 0.1, "learning_rate": 0.0007278475426278837, "loss": 1.6737, "theoretical_loss": 4.132474999624861, "tokens_seen": 327680000 } ], "max_steps": 50354, "num_train_epochs": 9223372036854775807, "total_flos": 1.6722690048e+17, "trial_name": null, "trial_params": null }